From 2a540206a74af8d38a01aaa5a37adc1008cad6ca Mon Sep 17 00:00:00 2001
From: nramirezuy <nramirez.uy@gmail.com>
Date: Tue, 19 Aug 2014 13:57:00 -0300
Subject: [PATCH 001/716] fix xmliter namespace on selected node

---
 scrapy/utils/iterators.py     | 22 +++++++++++++++-------
 tests/test_utils_iterators.py |  6 +++++-
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 150b077ae..11b873f2e 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -20,19 +20,27 @@ def xmliter(obj, nodename):
     - a unicode string
     - a string encoded as utf-8
     """
-    HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S)
+    DOCUMENT_HEADER_RE = re.compile(r'<\?xml[^>]+>\s*', re.S)
     HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S)
+    END_TAG_RE = re.compile(r'<\s*/([^\s>]+)\s*>', re.S)
+    NAMESPACE_RE = re.compile(r'((xmlns[:A-Za-z]*)=[^>\s]+)', re.S)
     text = _body_or_str(obj)
 
-    header_start = re.search(HEADER_START_RE, text)
-    header_start = header_start.group(1).strip() if header_start else ''
-    header_end = re_rsearch(HEADER_END_RE, text)
-    header_end = text[header_end[1]:].strip() if header_end else ''
+    document_header = re.search(DOCUMENT_HEADER_RE, text)
+    document_header = document_header.group().strip() if document_header else ''
+    header_end_idx = re_rsearch(HEADER_END_RE, text)
+    header_end = text[header_end_idx[1]:].strip() if header_end_idx else ''
+    namespaces = {}
+    if header_end:
+        for tagname in reversed(re.findall(END_TAG_RE, header_end)):
+            tag = re.search(r'<\s*%s.*?xmlns[:=][^>]*>' % tagname, text[:header_end_idx[1]], re.S)
+            if tag:
+                namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group()))
 
     r = re.compile(r"<%s[\s>].*?</%s>" % (nodename, nodename), re.DOTALL)
     for match in r.finditer(text):
-        nodetext = header_start + match.group() + header_end
-        yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
+        nodetext = document_header + match.group().replace(nodename, '%s %s' % (nodename, ' '.join(namespaces.values())), 1) + header_end
+        yield Selector(text=nodetext, type='xml')
 
 
 def csviter(obj, delimiter=None, headers=None, encoding=None):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index fe53f831f..8b5941605 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -61,7 +61,6 @@ class XmliterTestCase(unittest.TestCase):
         """
         response = XmlResponse(url='http://mydummycompany.com', body=body)
         my_iter = self.xmliter(response, 'item')
-
         node = next(my_iter)
         node.register_namespace('g', 'http://base.google.com/ns/1.0')
         self.assertEqual(node.xpath('title/text()').extract(), ['Item 1'])
@@ -74,6 +73,11 @@ class XmliterTestCase(unittest.TestCase):
         self.assertEqual(node.xpath('id/text()').extract(), [])
         self.assertEqual(node.xpath('price/text()').extract(), [])
 
+        my_iter = self.xmliter(response, 'g:image_link')
+        node = next(my_iter)
+        node.register_namespace('g', 'http://base.google.com/ns/1.0')
+        self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
+
     def test_xmliter_exception(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
 

From 114437c1693c85125c4275387616d398e68ec20f Mon Sep 17 00:00:00 2001
From: Pengyu CHEN <pengyu@libstarrify.so>
Date: Wed, 4 Nov 2015 02:29:28 +0800
Subject: [PATCH 002/716] added: Doc for `scrapy.http.TextResponse.urljoin`

---
 docs/topics/request-response.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 75b98d3b3..ba3d697ef 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -666,6 +666,14 @@ TextResponse objects
         The same as :attr:`text`, but available as a method. This method is
         kept for backwards compatibility; please prefer ``response.text``.
 
+    .. method:: TextResponse.urljoin(url)
+
+        Constructs an absolute url by combining the Response's base url with
+        a possible relative url. The base url shall be extracted from the
+        ``<base>`` tag, or just the Response's :attr:`url` if there is no such
+        tag.
+
+
 
 HtmlResponse objects
 --------------------

From 412f8526029f63fbebd6dae3bad7240dee8dc090 Mon Sep 17 00:00:00 2001
From: Arvind Prasanna <1108710+aprasanna@users.noreply.github.com>
Date: Tue, 6 Mar 2018 23:58:27 -0500
Subject: [PATCH 003/716] A few typo fixes and some grammatical enhancements

---
 docs/news.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 1629510b2..be4cac3f3 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1888,7 +1888,7 @@ Scrapy 0.14.3
 - include egg files used by testsuite in source distribution. #118 (:commit:`c897793`)
 - update docstring in project template to avoid confusion with genspider command, which may be considered as an advanced feature. refs #107 (:commit:`2548dcc`)
 - added note to docs/topics/firebug.rst about google directory being shut down (:commit:`668e352`)
-- dont discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`)
+- do not discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`)
 - do not fail handling unicode xpaths in libxml2 backed selectors (:commit:`b830e95`)
 - fixed minor mistake in Request objects documentation (:commit:`bf3c9ee`)
 - fixed minor defect in link extractors documentation (:commit:`ba14f38`)
@@ -1984,7 +1984,7 @@ Code rearranged and removed
 - Removed googledir project from `examples/googledir`. There's now a new example project called `dirbot` available on github: https://github.com/scrapy/dirbot
 - Removed support for default field values in Scrapy items (:rev:`2616`)
 - Removed experimental crawlspider v2 (:rev:`2632`)
-- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`)
+- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe filtering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`)
 - Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`)
 - Removed deprecated Execution Queue (:rev:`2704`)
 - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`)
@@ -2054,7 +2054,7 @@ New features and improvements
 - Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195)
 - Support for overriding default request headers per spider (#181)
 - Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186)
-- Splitted Debian package into two packages - the library and the service (#187)
+- The Debian package has been split into two packages - library and the service (#187)
 - Scrapy log refactoring (#188)
 - New extension for keeping persistent spider contexts among different runs (#203)
 - Added `dont_redirect` request.meta key for avoiding redirects (#233)
@@ -2075,7 +2075,7 @@ API changes
 - ``url`` and ``body`` attributes of Request objects are now read-only (#230)
 - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
 - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
-- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
+- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
 - Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself.
 - Changes to Scrapy Manager (now called "Crawler"):
    - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``

From ac111088c6470d55e4d698247a76a4938f4a6dc6 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sun, 13 Jan 2019 20:12:29 +0530
Subject: [PATCH 004/716] duplicate keys handled

---
 scrapy/http/request/form.py | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index c2413b431..660f4fc96 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst
 """
 
 import six
-from six.moves.urllib.parse import urljoin, urlencode
+from six.moves.urllib.parse import urljoin, urlencode, urlsplit
 
 import lxml.html
 from parsel.selector import create_root_node
@@ -33,7 +33,30 @@ class FormRequest(Request):
                 self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
                 self._set_body(querystr)
             else:
-                self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr)
+                if urlsplit(self.url).query:
+                    queries = (urlsplit(self.url).query + '&' + querystr).split('&')
+                else:
+                    queries = querystr.split('&')
+                query_dict = {}
+                for i in range(len(queries)):
+                    query_list = queries[i].split('=')
+                    query_dict[query_list[0]] = query_list[1]
+                querystr = ''
+                query_key = list(query_dict.keys())
+                for i in range(len(query_dict)):
+                    querystr += (query_key[i] + '=' + query_dict[query_key[i]])
+                    if i!=len(query_dict)-1:
+                        querystr += '&'
+                if urlsplit(self.url).fragment:
+                    if urlsplit(self.url).query:
+                        self._set_url(self.url[:self.url.index('?')+1]+querystr+'#'+urlsplit(self.url).fragment)
+                    else:
+                        self._set_url(self.url+'?'+querystr+'#'+urlsplit(self.url).fragment)
+                else:
+                    if urlsplit(self.url).query:
+                        self._set_url(self.url[:self.url.index('?')+1]+querystr)
+                    else:
+                        self._set_url(self.url+'?'+querystr)
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From b5e454809e240839e76f050b4587abcd80c33765 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sun, 13 Jan 2019 20:12:31 +0530
Subject: [PATCH 005/716] Included test

---
 tests/test_http_request.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 58326a384..5a7cf3918 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -273,6 +273,19 @@ class FormRequestTest(RequestTest):
     def test_empty_formdata(self):
         r1 = self.request_class("http://www.example.com", formdata={})
         self.assertEqual(r1.body, b'')
+        
+    def test_formdata_overrides_querystring_duplicates(self):
+        data1 = {'a' : '1', 'b' : '2'}
+        data2 = [('a','one'), ('a','two'), ('c','three')]
+    
+        fs1 = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1))
+        self.assertEqual(fs1[b'a'], [b'1'])
+        self.assertEqual(fs1[b'b'], [b'2'])
+
+        fs2 = _qs(self.request_class('http://www.example.com?a=1&b=2&c=3', method='GET', formdata=data2))
+        self.assertEqual(fs2[b'a'], [b'two'])
+        self.assertEqual(fs2[b'b'], [b'2'])
+        self.assertEqual(fs2[b'c'], [b'three'])
 
     def test_default_encoding_bytes(self):
         # using default encoding (utf-8)

From 1bea5d307628a11d4010cdabb87b2a29a6c3c772 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sun, 13 Jan 2019 22:35:16 +0530
Subject: [PATCH 006/716] Fixed error

---
 scrapy/http/request/form.py | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 660f4fc96..f08d1f626 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -34,29 +34,32 @@ class FormRequest(Request):
                 self._set_body(querystr)
             else:
                 if urlsplit(self.url).query:
-                    queries = (urlsplit(self.url).query + '&' + querystr).split('&')
+                    queries = (urlsplit(self.url).query).split('&')
                 else:
                     queries = querystr.split('&')
                 query_dict = {}
+                duplicate_key_passed=[]
+                for i in range(len(querystr.split('&'))):
+                    query_list = querystr.split('&')[i].split('=')
+                    duplicate_key_passed.append(query_list[0])
                 for i in range(len(queries)):
                     query_list = queries[i].split('=')
-                    query_dict[query_list[0]] = query_list[1]
-                querystr = ''
+                    if duplicate_key_passed.count(query_list[0])<=1:
+                        query_dict[query_list[0]] = query_list[1]  
+                query_str = ''
                 query_key = list(query_dict.keys())
                 for i in range(len(query_dict)):
-                    querystr += (query_key[i] + '=' + query_dict[query_key[i]])
-                    if i!=len(query_dict)-1:
-                        querystr += '&'
+                    query_str += (query_key[i] + '=' + query_dict[query_key[i]] + '&')       
                 if urlsplit(self.url).fragment:
                     if urlsplit(self.url).query:
-                        self._set_url(self.url[:self.url.index('?')+1]+querystr+'#'+urlsplit(self.url).fragment)
+                        self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + '#' + urlsplit(self.url).fragment)
                     else:
-                        self._set_url(self.url+'?'+querystr+'#'+urlsplit(self.url).fragment)
+                        self._set_url(self.url + '?' + querystr + '#' + urlsplit(self.url).fragment)
                 else:
                     if urlsplit(self.url).query:
-                        self._set_url(self.url[:self.url.index('?')+1]+querystr)
+                        self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr)
                     else:
-                        self._set_url(self.url+'?'+querystr)
+                        self._set_url(self.url + '?' + querystr)
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From 9a4bbd6d029f61dc60594fb8bca631c3a494f855 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sun, 13 Jan 2019 23:05:58 +0530
Subject: [PATCH 007/716] Update form.py

---
 scrapy/http/request/form.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index f08d1f626..1b487cbd0 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -44,7 +44,7 @@ class FormRequest(Request):
                     duplicate_key_passed.append(query_list[0])
                 for i in range(len(queries)):
                     query_list = queries[i].split('=')
-                    if duplicate_key_passed.count(query_list[0])<=1:
+                    if duplicate_key_passed.count(query_list[0])==0:
                         query_dict[query_list[0]] = query_list[1]  
                 query_str = ''
                 query_key = list(query_dict.keys())

From 4abcdcb306d8782e6b792ad36e40391a0bcbd0f3 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sun, 13 Jan 2019 23:22:53 +0530
Subject: [PATCH 008/716] Update test_http_request.py

---
 tests/test_http_request.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 5a7cf3918..ec247a02d 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -276,17 +276,10 @@ class FormRequestTest(RequestTest):
         
     def test_formdata_overrides_querystring_duplicates(self):
         data1 = {'a' : '1', 'b' : '2'}
-        data2 = [('a','one'), ('a','two'), ('c','three')]
-    
-        fs1 = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1))
+        fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1))
         self.assertEqual(fs1[b'a'], [b'1'])
         self.assertEqual(fs1[b'b'], [b'2'])
 
-        fs2 = _qs(self.request_class('http://www.example.com?a=1&b=2&c=3', method='GET', formdata=data2))
-        self.assertEqual(fs2[b'a'], [b'two'])
-        self.assertEqual(fs2[b'b'], [b'2'])
-        self.assertEqual(fs2[b'c'], [b'three'])
-
     def test_default_encoding_bytes(self):
         # using default encoding (utf-8)
         data = {b'one': b'two', b'price': b'\xc2\xa3 100'}

From 023290dabc55c92ea2143a98f7753abaf21f74c9 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sun, 13 Jan 2019 23:50:31 +0530
Subject: [PATCH 009/716] Update test_http_request.py

---
 tests/test_http_request.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index ec247a02d..1ef84f1d6 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -275,10 +275,10 @@ class FormRequestTest(RequestTest):
         self.assertEqual(r1.body, b'')
         
     def test_formdata_overrides_querystring_duplicates(self):
-        data1 = {'a' : '1', 'b' : '2'}
-        fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1))
-        self.assertEqual(fs1[b'a'], [b'1'])
-        self.assertEqual(fs1[b'b'], [b'2'])
+        data = {'a' : '1', 'b' : '2'}
+        fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data))
+        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(fs[b'b'], [b'2'])
 
     def test_default_encoding_bytes(self):
         # using default encoding (utf-8)

From 9f1f4df9666f38f81adda5a41f3cac646a4f18dc Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Wed, 16 Jan 2019 22:59:41 +0530
Subject: [PATCH 010/716] Update test_http_request.py

---
 tests/test_http_request.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 1ef84f1d6..6c4cb10a8 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -275,8 +275,22 @@ class FormRequestTest(RequestTest):
         self.assertEqual(r1.body, b'')
         
     def test_formdata_overrides_querystring_duplicates(self):
+        #Without fragment
         data = {'a' : '1', 'b' : '2'}
-        fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data))
+        fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data))
+        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(fs[b'b'], [b'2'])
+
+        #With fragment
+        data = (('a', '1'), ('b', '2'))
+        url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0]
+        fs = _qs(self.request_class(url, method='GET', formdata=data))
+        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(fs[b'b'], [b'2'])
+
+        #Witout query
+        data = {'a' : '1', 'b' : '2'}
+        fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])
 

From 3e67fa8fc1eaa2189cec08ce4000b3efc254ee1b Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Wed, 16 Jan 2019 23:01:38 +0530
Subject: [PATCH 011/716] Improved for better user readability

---
 scrapy/http/request/form.py | 45 ++++++++++++++++---------------------
 1 file changed, 19 insertions(+), 26 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 1b487cbd0..fa5110382 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -33,33 +33,26 @@ class FormRequest(Request):
                 self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
                 self._set_body(querystr)
             else:
-                if urlsplit(self.url).query:
-                    queries = (urlsplit(self.url).query).split('&')
+                url_split = urlsplit(self.url)
+                formdata_key_list = []
+                for k in querystr.split('&'):
+                    formdata_key_list.append(k.split('=')[0])
+
+                if url_split.query:
+                    queries = (url_split.query).split('&')
+                    query_dict = {}
+
+                    for x in queries:
+                        query = x.split('=')
+                        if formdata_key_list.count(query[0])==0:
+                            query_dict[query[0]] = query[1]
+                        query_str = ''
+                        for k, v in query_dict:
+                            query_str += (k + '=' + v + '&')
+
+                    self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else ''))
                 else:
-                    queries = querystr.split('&')
-                query_dict = {}
-                duplicate_key_passed=[]
-                for i in range(len(querystr.split('&'))):
-                    query_list = querystr.split('&')[i].split('=')
-                    duplicate_key_passed.append(query_list[0])
-                for i in range(len(queries)):
-                    query_list = queries[i].split('=')
-                    if duplicate_key_passed.count(query_list[0])==0:
-                        query_dict[query_list[0]] = query_list[1]  
-                query_str = ''
-                query_key = list(query_dict.keys())
-                for i in range(len(query_dict)):
-                    query_str += (query_key[i] + '=' + query_dict[query_key[i]] + '&')       
-                if urlsplit(self.url).fragment:
-                    if urlsplit(self.url).query:
-                        self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + '#' + urlsplit(self.url).fragment)
-                    else:
-                        self._set_url(self.url + '?' + querystr + '#' + urlsplit(self.url).fragment)
-                else:
-                    if urlsplit(self.url).query:
-                        self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr)
-                    else:
-                        self._set_url(self.url + '?' + querystr)
+                    self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else ''))
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From 6f86c93f366509ba05faf3b2cdc12f5e3ef6693a Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Wed, 16 Jan 2019 23:54:35 +0530
Subject: [PATCH 012/716] Increased test cases

---
 tests/test_http_request.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 6c4cb10a8..571df0176 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -275,20 +275,27 @@ class FormRequestTest(RequestTest):
         self.assertEqual(r1.body, b'')
         
     def test_formdata_overrides_querystring_duplicates(self):
-        #Without fragment
-        data = {'a' : '1', 'b' : '2'}
-        fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data))
-        self.assertEqual(fs[b'a'], [b'1'])
-        self.assertEqual(fs[b'b'], [b'2'])
-
-        #With fragment
+        #Both fragment and query in url
         data = (('a', '1'), ('b', '2'))
         url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0]
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])
 
-        #Witout query
+        #only query in url
+        data = {'a' : '1', 'b' : '2'}
+        fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data))
+        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(fs[b'b'], [b'2'])
+
+        #only fragment in url
+        data = (('a', '1'), ('b', '2'))
+        url = self.request_class('http://www.example.com/#fragment', method='GET', formdata=data).url.split('#')[0]
+        fs = _qs(self.request_class(url, method='GET', formdata=data))
+        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(fs[b'b'], [b'2'])
+
+        #None of both in url
         data = {'a' : '1', 'b' : '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))
         self.assertEqual(fs[b'a'], [b'1'])

From 6be73f06c33a27cb3291a1e46d80574b4d23b10b Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Thu, 17 Jan 2019 23:50:58 +0530
Subject: [PATCH 013/716] Updated tests

---
 tests/test_http_request.py | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 571df0176..a17acf93b 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -277,23 +277,11 @@ class FormRequestTest(RequestTest):
     def test_formdata_overrides_querystring_duplicates(self):
         #Both fragment and query in url
         data = (('a', '1'), ('b', '2'))
-        url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0]
-        fs = _qs(self.request_class(url, method='GET', formdata=data))
-        self.assertEqual(fs[b'a'], [b'1'])
-        self.assertEqual(fs[b'b'], [b'2'])
-
-        #only query in url
-        data = {'a' : '1', 'b' : '2'}
-        fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data))
-        self.assertEqual(fs[b'a'], [b'1'])
-        self.assertEqual(fs[b'b'], [b'2'])
-
-        #only fragment in url
-        data = (('a', '1'), ('b', '2'))
-        url = self.request_class('http://www.example.com/#fragment', method='GET', formdata=data).url.split('#')[0]
+        url = self.request_class('http://www.example.com/?a=0&a=2&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])
+        self.assertEqual(fs[b'c'], [b'3'])
 
         #None of both in url
         data = {'a' : '1', 'b' : '2'}

From a9f68acb6dd7f5aca2bee95353278e5a40c32b8b Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Thu, 17 Jan 2019 23:51:09 +0530
Subject: [PATCH 014/716] modified code

---
 scrapy/http/request/form.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index fa5110382..2305491e2 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -41,15 +41,14 @@ class FormRequest(Request):
                 if url_split.query:
                     queries = (url_split.query).split('&')
                     query_dict = {}
-
                     for x in queries:
-                        query = x.split('=')
-                        if formdata_key_list.count(query[0])==0:
-                            query_dict[query[0]] = query[1]
+                        k = x.split('=')[0]
+                        v = x.split('=')[1]
+                        if formdata_key_list.count(k)==0:
+                            query_dict[k] = v
                         query_str = ''
-                        for k, v in query_dict:
+                        for k, v in query_dict.items():
                             query_str += (k + '=' + v + '&')
-
                     self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else ''))
                 else:
                     self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else ''))

From 7dee841b8b904a3961481f7b1999d819d2c17c5d Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sat, 19 Jan 2019 13:20:01 +0530
Subject: [PATCH 015/716] Update form.py

---
 scrapy/http/request/form.py | 21 +++++----------------
 1 file changed, 5 insertions(+), 16 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 2305491e2..af4ed1793 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst
 """
 
 import six
-from six.moves.urllib.parse import urljoin, urlencode, urlsplit
+from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl
 
 import lxml.html
 from parsel.selector import create_root_node
@@ -37,21 +37,10 @@ class FormRequest(Request):
                 formdata_key_list = []
                 for k in querystr.split('&'):
                     formdata_key_list.append(k.split('=')[0])
-
-                if url_split.query:
-                    queries = (url_split.query).split('&')
-                    query_dict = {}
-                    for x in queries:
-                        k = x.split('=')[0]
-                        v = x.split('=')[1]
-                        if formdata_key_list.count(k)==0:
-                            query_dict[k] = v
-                        query_str = ''
-                        for k, v in query_dict.items():
-                            query_str += (k + '=' + v + '&')
-                    self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else ''))
-                else:
-                    self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else ''))
+                items = []
+                items += [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list]
+                query_str = _urlencode(items, self.encoding)
+                self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else '')))
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From 722a30ac2bc35bdc7abb52d10417ed1191a3aee0 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com>
Date: Sat, 19 Jan 2019 13:20:05 +0530
Subject: [PATCH 016/716] Update test_http_request.py

---
 tests/test_http_request.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index a17acf93b..955db9e76 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -276,10 +276,10 @@ class FormRequestTest(RequestTest):
         
     def test_formdata_overrides_querystring_duplicates(self):
         #Both fragment and query in url
-        data = (('a', '1'), ('b', '2'))
-        url = self.request_class('http://www.example.com/?a=0&a=2&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
+        data = (('a', 'one'), ('a', 'two'), ('b', '2'))
+        url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
         fs = _qs(self.request_class(url, method='GET', formdata=data))
-        self.assertEqual(fs[b'a'], [b'1'])
+        self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])
         self.assertEqual(fs[b'c'], [b'3'])
 

From 6eca6f92c6ae48bb136311308e77e82d7659cf43 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Mon, 4 Mar 2019 14:59:34 +0530
Subject: [PATCH 017/716] Update form.py

---
 scrapy/http/request/form.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index af4ed1793..b6ca6ef9c 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -34,11 +34,8 @@ class FormRequest(Request):
                 self._set_body(querystr)
             else:
                 url_split = urlsplit(self.url)
-                formdata_key_list = []
-                for k in querystr.split('&'):
-                    formdata_key_list.append(k.split('=')[0])
-                items = []
-                items += [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list]
+                formdata_key_list = list(dict(parse_qsl(querystr)).keys())
+                items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list]
                 query_str = _urlencode(items, self.encoding)
                 self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else '')))
 

From d75b61b96aad172e21c7d3aeba6f48419a63c72d Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Mon, 4 Mar 2019 15:07:12 +0530
Subject: [PATCH 018/716] Update test_http_request.py

---
 tests/test_http_request.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 955db9e76..2da1cdf6d 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -289,6 +289,12 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])
 
+        #GET duplicates are preserved
+        formdata=(('foo', 'bar'), ('foo', 'baz'))
+        fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data))
+        self.assertEqual(set(fs[b'foo']), {b'bar', b'baz'})
+        self.assertEqual(set(fs[b'a']), {b'1', b'2'})
+
     def test_default_encoding_bytes(self):
         # using default encoding (utf-8)
         data = {b'one': b'two', b'price': b'\xc2\xa3 100'}

From fdf03a6d0daae79b9a3b421d19102873d5a75f46 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Mon, 4 Mar 2019 15:12:44 +0530
Subject: [PATCH 019/716] correcting tests

---
 tests/test_http_request.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 2da1cdf6d..0c2f95262 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -289,10 +289,10 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])
 
-        #GET duplicates are preserved
-        formdata=(('foo', 'bar'), ('foo', 'baz'))
+        #Duplicate GET arguments are preserved
+        formdata={'foo' : 'bar'}
         fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data))
-        self.assertEqual(set(fs[b'foo']), {b'bar', b'baz'})
+        self.assertEqual(fs[b'foo'], [b'bar'])
         self.assertEqual(set(fs[b'a']), {b'1', b'2'})
 
     def test_default_encoding_bytes(self):

From 8831fafabc699bd2ab48e3e185712fbcd4f7cc7f Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Mon, 4 Mar 2019 15:42:48 +0530
Subject: [PATCH 020/716] Update test_http_request.py

---
 tests/test_http_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 0c2f95262..5c30c9785 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -290,7 +290,7 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[b'b'], [b'2'])
 
         #Duplicate GET arguments are preserved
-        formdata={'foo' : 'bar'}
+        data={'foo' : 'bar'}
         fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data))
         self.assertEqual(fs[b'foo'], [b'bar'])
         self.assertEqual(set(fs[b'a']), {b'1', b'2'})

From 7da460b7935940c93d5454019ea99f9f117b3e7b Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Mon, 4 Mar 2019 17:25:15 +0530
Subject: [PATCH 021/716] Update form.py

---
 scrapy/http/request/form.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index b6ca6ef9c..b44d7b9a6 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst
 """
 
 import six
-from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl
+from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl, urlunsplit
 
 import lxml.html
 from parsel.selector import create_root_node
@@ -34,10 +34,10 @@ class FormRequest(Request):
                 self._set_body(querystr)
             else:
                 url_split = urlsplit(self.url)
-                formdata_key_list = list(dict(parse_qsl(querystr)).keys())
-                items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list]
+                formdata_keys = set(dict(parse_qsl(querystr)).keys())
+                items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys]
                 query_str = _urlencode(items, self.encoding)
-                self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else '')))
+                self._set_url(urlunsplit((url_split.scheme, url_split.netloc, url_split.path, (query_str + '&' + querystr) if query_str else querystr, url_split.fragment)))
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From f7bf3abbd03a10124ed648d45abbb6c8eb3218b7 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Wed, 6 Mar 2019 14:10:03 +0530
Subject: [PATCH 022/716] Modified code

---
 scrapy/http/request/form.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index b44d7b9a6..a0e7f76a9 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -37,7 +37,8 @@ class FormRequest(Request):
                 formdata_keys = set(dict(parse_qsl(querystr)).keys())
                 items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys]
                 query_str = _urlencode(items, self.encoding)
-                self._set_url(urlunsplit((url_split.scheme, url_split.netloc, url_split.path, (query_str + '&' + querystr) if query_str else querystr, url_split.fragment)))
+                query = (query_str + '&' + querystr) if query_str else querystr
+                self._set_url(urlunsplit(url_split._replace(query = query)))
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From 120007c0577f819a26ff795ac8b9cd6fc397df19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 8 Mar 2019 13:53:47 +0100
Subject: [PATCH 023/716] Add a FAQ entry on how to deal with long lists of
 allowed domains

---
 docs/faq.rst                      | 35 +++++++++++++++++++++++++++++++
 docs/topics/spider-middleware.rst |  2 ++
 2 files changed, 37 insertions(+)

diff --git a/docs/faq.rst b/docs/faq.rst
index 7a0628f88..f56d26c0a 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -149,6 +149,41 @@ How can I make Scrapy consume less memory?
 
 See previous question.
 
+How can I prevent memory errors due to many allowed domains?
+------------------------------------------------------------
+
+If you have a spider with a long list of
+:attr:`~scrapy.spiders.Spider.allowed_domains` (e.g. 50,000+), consider
+replacing the default
+:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware
+with a :ref:`custom spider middleware <custom-spider-middleware>` that requires
+less memory. For example:
+
+-   If your domain names are similar enough, use your own regular expression
+    instead joining the strings in
+    :attr:`~scrapy.spiders.Spider.allowed_domains` into a complex regular
+    expression.
+
+-   If you can `meet the installation requirements`_, use pyre2_ instead of
+    Python’s re_ to compile your URL-filtering regular expression. See
+    :issue:`1908`.
+
+See also other suggestions at `StackOverflow`_.
+
+.. note:: Remember to disable
+   :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable
+   your custom implementation::
+
+       SPIDER_MIDDLEWARES = {
+           'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
+           'myproject.middlewares.CustomOffsiteMiddleware': 500,
+       }
+
+.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation
+.. _pyre2: https://github.com/andreasvc/pyre2
+.. _re: https://docs.python.org/library/re.html
+.. _StackOverflow: https://stackoverflow.com/q/36440681/939364
+
 Can I use Basic HTTP Authentication in my spiders?
 --------------------------------------------------
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 2b7e42771..80357a987 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -54,6 +54,8 @@ value.  For example, if you want to disable the off-site middleware::
 Finally, keep in mind that some middlewares may need to be enabled through a
 particular setting. See each middleware documentation for more info.
 
+.. _custom-spider-middleware:
+
 Writing your own spider middleware
 ==================================
 

From 35f7595dbebe7012c8f66336b76625b089a1fad3 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Mon, 11 Mar 2019 23:58:37 +0530
Subject: [PATCH 024/716] changed variable names

---
 scrapy/http/request/form.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index a0e7f76a9..9af2db5ff 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -28,16 +28,16 @@ class FormRequest(Request):
 
         if formdata:
             items = formdata.items() if isinstance(formdata, dict) else formdata
-            querystr = _urlencode(items, self.encoding)
+            form_query_str = _urlencode(items, self.encoding)
             if self.method == 'POST':
                 self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
-                self._set_body(querystr)
+                self._set_body(form_query_str)
             else:
                 url_split = urlsplit(self.url)
-                formdata_keys = set(dict(parse_qsl(querystr)).keys())
+                formdata_keys = set(dict(parse_qsl(form_query_str)).keys())
                 items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys]
-                query_str = _urlencode(items, self.encoding)
-                query = (query_str + '&' + querystr) if query_str else querystr
+                url_query_str = _urlencode(items, self.encoding)
+                query = (url_query_str + '&' + form_query_str) if url_query_str else form_query_str
                 self._set_url(urlunsplit(url_split._replace(query = query)))
 
     @classmethod

From 282f24c510bdc6fff0e86ca16b7d681617af25ca Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Wed, 20 Mar 2019 18:46:22 +0530
Subject: [PATCH 025/716] Update form.py

---
 scrapy/http/request/form.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 9af2db5ff..03692f063 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst
 """
 
 import six
-from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl, urlunsplit
+from six.moves.urllib.parse import urljoin, urlencode, urlsplit, urlunsplit
 
 import lxml.html
 from parsel.selector import create_root_node
@@ -33,12 +33,7 @@ class FormRequest(Request):
                 self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
                 self._set_body(form_query_str)
             else:
-                url_split = urlsplit(self.url)
-                formdata_keys = set(dict(parse_qsl(form_query_str)).keys())
-                items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys]
-                url_query_str = _urlencode(items, self.encoding)
-                query = (url_query_str + '&' + form_query_str) if url_query_str else form_query_str
-                self._set_url(urlunsplit(url_split._replace(query = query)))
+                self._set_url(urlunsplit(urlsplit(self.url)._replace(query = form_query_str)))
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,

From 4c89e53e684987c61857ef810374101440d5616c Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Wed, 20 Mar 2019 18:46:25 +0530
Subject: [PATCH 026/716] Update test_http_request.py

---
 tests/test_http_request.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 5c30c9785..feea54a54 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -275,26 +275,17 @@ class FormRequestTest(RequestTest):
         self.assertEqual(r1.body, b'')
         
     def test_formdata_overrides_querystring_duplicates(self):
-        #Both fragment and query in url
         data = (('a', 'one'), ('a', 'two'), ('b', '2'))
         url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])
-        self.assertEqual(fs[b'c'], [b'3'])
 
-        #None of both in url
         data = {'a' : '1', 'b' : '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])
 
-        #Duplicate GET arguments are preserved
-        data={'foo' : 'bar'}
-        fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data))
-        self.assertEqual(fs[b'foo'], [b'bar'])
-        self.assertEqual(set(fs[b'a']), {b'1', b'2'})
-
     def test_default_encoding_bytes(self):
         # using default encoding (utf-8)
         data = {b'one': b'two', b'price': b'\xc2\xa3 100'}

From 72cf190145196c3054b611eef7a0eef30ac63c8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 8 Mar 2019 14:46:07 +0100
Subject: [PATCH 027/716] Add a FAQ entry about name collisions

---
 docs/faq.rst | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 7a0628f88..8de680816 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -319,7 +319,18 @@ I'm scraping a XML document and my XPath selector doesn't return any items
 
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 
+Running ``runspider`` I get ``error: No spider found in file: <filename>``
+--------------------------------------------------------------------------
+
+This may happen if your Scrapy project has a spider module with a name that
+conflicts with the name of one of the `Python standard library modules`_, such
+as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed.
+See :issue:`2680`.
+
+.. _Python standard library modules: https://docs.python.org/py-modindex.html
+.. _Python package: https://pypi.org/
+
 .. _user agents: https://en.wikipedia.org/wiki/User_agent
 .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
 .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
-.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search
+.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search
\ No newline at end of file

From dc8310e2929d228a224f12055a26cfef71d751cd Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Tue, 26 Mar 2019 15:42:58 +0530
Subject: [PATCH 028/716] changed tests

---
 tests/test_http_request.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index feea54a54..81c1a4a9e 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -274,12 +274,13 @@ class FormRequestTest(RequestTest):
         r1 = self.request_class("http://www.example.com", formdata={})
         self.assertEqual(r1.body, b'')
         
-    def test_formdata_overrides_querystring_duplicates(self):
+    def test_formdata_overrides_querystring(self):
         data = (('a', 'one'), ('a', 'two'), ('b', '2'))
         url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])
+        self.assertNone(fs[b'c'])
 
         data = {'a' : '1', 'b' : '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))

From 213b9eb879c4c6d8ae60ffa950688a38951b3349 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Tue, 26 Mar 2019 15:59:38 +0530
Subject: [PATCH 029/716] Update test_http_request.py

---
 tests/test_http_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 81c1a4a9e..56c7d3d94 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -280,7 +280,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])
-        self.assertNone(fs[b'c'])
+        self.assertIsNone(fs[b'c'])
 
         data = {'a' : '1', 'b' : '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))

From ae856e8ba835d1d99510e9509a462b55a757e85f Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Tue, 26 Mar 2019 16:21:52 +0530
Subject: [PATCH 030/716] corrected tests

---
 tests/test_http_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 56c7d3d94..55fa4ad22 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -280,7 +280,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])
-        self.assertIsNone(fs[b'c'])
+        self.assertIsNone(fs.get([b'c']))
 
         data = {'a' : '1', 'b' : '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))

From 5f2ad5377e54ecdb3059891db4887eb09e56bfc6 Mon Sep 17 00:00:00 2001
From: Maram Sumanth <maram.sumanth@gmail.com>
Date: Tue, 26 Mar 2019 16:46:15 +0530
Subject: [PATCH 031/716] fixed typo

---
 tests/test_http_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 55fa4ad22..89e4a8683 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -280,7 +280,7 @@ class FormRequestTest(RequestTest):
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])
-        self.assertIsNone(fs.get([b'c']))
+        self.assertIsNone(fs.get(b'c'))
 
         data = {'a' : '1', 'b' : '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))

From 484bd0d22a11a04ab775ac4f72c75f3ec9050d98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 22 Mar 2019 16:59:29 +0100
Subject: [PATCH 032/716] Allow customizing export column names

---
 docs/topics/exporters.rst       | 35 +++++++++++++++-----
 docs/topics/feed-exports.rst    | 17 +++-------
 scrapy/exporters.py             | 32 +++++++++++++++---
 scrapy/extensions/feedexport.py |  3 +-
 scrapy/settings/__init__.py     | 35 +++++++++++++++++++-
 tests/test_exporters.py         | 15 ++++++++-
 tests/test_feedexport.py        | 57 +++++++++++++++++++++++++++++++++
 7 files changed, 165 insertions(+), 29 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index f5048d2da..42a93e459 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -190,14 +190,33 @@ BaseItemExporter
 
    .. attribute:: fields_to_export
 
-      A list with the name of the fields that will be exported, or None if you
-      want to export all fields. Defaults to None.
+      Fields to export, their order [1]_ and their output names.
 
-      Some exporters (like :class:`CsvItemExporter`) respect the order of the
-      fields defined in this attribute.
+      Possible values are:
 
-      Some exporters may require fields_to_export list in order to export the
-      data properly when spiders return dicts (not :class:`~Item` instances).
+      -   ``None`` (all fields [2]_, default)
+
+      -   A list of fields::
+
+              ['field1', 'field2']
+
+      -   A dict [3]_ where keys are fields and values are output names::
+
+              {'field1': 'Field 1', 'field2': 'Field 2'}
+
+      .. [1] Not all exporters respect the specified field order.
+      .. [2] If you yield items as dicts (not :class:`Item` instances),
+             exporters that need to know the fields to export beforehand, like
+             :class:`CsvItemExporter`, only export the fields found in the
+             first item.
+      .. [3] Dicts preserve insertion order since `Python 3.7`_
+             (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version
+             of Python, use an OrderedDict_ to enforce a specific field order.
+
+             .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html
+             .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation
+             .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html
+             .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict
 
    .. attribute:: export_empty_fields
 
@@ -286,8 +305,8 @@ CsvItemExporter
 
    Exports Items in CSV format to the given file-like object. If the
    :attr:`fields_to_export` attribute is set, it will be used to define the
-   CSV columns and their order. The :attr:`export_empty_fields` attribute has
-   no effect on this exporter.
+   CSV columns, their order and their column names. The
+   :attr:`export_empty_fields` attribute has no effect on this exporter.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index cf70b8aca..968cb8884 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -56,7 +56,7 @@ CSV
 
  * :setting:`FEED_FORMAT`: ``csv``
  * Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
- * To specify columns to export and their order use
+ * To specify columns to export, their order and their column names, use
    :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
    option, but it is important for CSV because unlike many other export
    formats CSV uses a fixed header.
@@ -259,18 +259,9 @@ FEED_EXPORT_FIELDS
 
 Default: ``None``
 
-A list of fields to export, optional.
-Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
-
-Use FEED_EXPORT_FIELDS option to define fields to export and their order.
-
-When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields
-defined in dicts or :class:`~.Item` subclasses a spider is yielding.
-
-If an exporter requires a fixed set of fields (this is the case for
-:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
-is empty or None, then Scrapy tries to infer field names from the
-exported data - currently it uses field names from the first item.
+Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their
+order and their output names. See :attr:`BaseItemExporter.fields_to_export
+<scrapy.exporters.BaseItemExporter.fields_to_export>` for more information.
 
 .. setting:: FEED_EXPORT_INDENT
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 695c74fec..c05acaca5 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -2,6 +2,7 @@
 Item Exporters are used to export/serialize items into different formats.
 """
 
+from collections import Mapping
 import csv
 import io
 import sys
@@ -64,6 +65,14 @@ class BaseItemExporter(object):
                 field_iter = six.iterkeys(item.fields)
             else:
                 field_iter = six.iterkeys(item)
+        elif isinstance(self.fields_to_export, Mapping):
+            if include_empty:
+                field_iter = self.fields_to_export.items()
+            else:
+                field_iter = (
+                    (x, y) for x, y in self.fields_to_export.items()
+                    if x in item
+                )
         else:
             if include_empty:
                 field_iter = self.fields_to_export
@@ -71,13 +80,22 @@ class BaseItemExporter(object):
                 field_iter = (x for x in self.fields_to_export if x in item)
 
         for field_name in field_iter:
-            if field_name in item:
-                field = {} if isinstance(item, dict) else item.fields[field_name]
-                value = self.serialize_field(field, field_name, item[field_name])
+            if isinstance(field_name, six.string_types):
+                item_field, output_field = field_name, field_name
+            else:
+                item_field, output_field = field_name
+            if item_field in item:
+                if isinstance(item, dict):
+                    field = {}
+                else:
+                    field = item.fields[item_field]
+                value = self.serialize_field(
+                    field, output_field, item[item_field]
+                )
             else:
                 value = default_value
 
-            yield field_name, value
+            yield output_field, value
 
 
 class JsonLinesItemExporter(BaseItemExporter):
@@ -259,7 +277,11 @@ class CsvItemExporter(BaseItemExporter):
                 else:
                     # use fields declared in Item
                     self.fields_to_export = list(item.fields.keys())
-            row = list(self._build_row(self.fields_to_export))
+            if isinstance(self.fields_to_export, Mapping):
+                fields = self.fields_to_export.values()
+            else:
+                fields = self.fields_to_export
+            row = list(self._build_row(fields))
             self.csv_writer.writerow(row)
 
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index b2f7267a2..1ed476d83 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -201,7 +201,8 @@ class FeedExporter(object):
             raise NotConfigured
         self.store_empty = settings.getbool('FEED_STORE_EMPTY')
         self._exporting = False
-        self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
+        self.export_fields = settings.getdictorlist('FEED_EXPORT_FIELDS')
+        self.export_fields = self.export_fields or None
         self.indent = None
         if settings.get('FEED_EXPORT_INDENT') is not None:
             self.indent = settings.getint('FEED_EXPORT_INDENT')
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 14c93bef2..69b324e84 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,7 +1,7 @@
 import six
 import json
 import copy
-from collections import MutableMapping
+from collections import MutableMapping, OrderedDict
 from importlib import import_module
 from pprint import pformat
 
@@ -198,6 +198,39 @@ class BaseSettings(MutableMapping):
             value = json.loads(value)
         return dict(value)
 
+    def getdictorlist(self, name, default=None):
+        """Get a setting value as either an ``OrderedDict`` or a list.
+
+        If the setting is already a dict or a list, a copy of it will be
+        returned.
+
+        If it is a string it will be evaluated as JSON, or as a comma-separated
+        list of strings as a fallback.
+
+        For example, settings populated through environment variables will
+        return:
+
+        -   ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to
+            ``'{"key1": "value1", "key2": "value2"}'``
+
+        -   ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
+
+        :param name: the setting name
+        :type name: string
+
+        :param default: the value to return if no setting is found
+        :type default: any
+        """
+        value = self.get(name, default)
+        if value is None:
+            return {}
+        if isinstance(value, six.string_types):
+            try:
+                return json.loads(value, object_pairs_hook=OrderedDict)
+            except ValueError:
+                return value.split(',')
+        return copy.deepcopy(value)
+
     def getwithbase(self, name):
         """Get a composition of a dictionary-like setting and its `_BASE`
         counterpart.
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index cd72c661a..1b3dc14a1 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -1,4 +1,7 @@
+# -*- coding:utf-8 -*-
+
 from __future__ import absolute_import
+from collections import OrderedDict
 import re
 import json
 import marshal
@@ -83,6 +86,14 @@ class BaseItemExporterTest(unittest.TestCase):
         assert isinstance(name, six.text_type)
         self.assertEqual(name, u'John\xa3')
 
+        ie = self._get_exporter(
+            fields_to_export=OrderedDict([('name', u'名稱')])
+        )
+        self.assertEqual(
+            list(ie._get_serialized_fields(self.i)),
+            [(u'名稱', u'John\xa3')]
+        )
+
     def test_field_custom_serializer(self):
         def custom_serializer(value):
             return str(int(value) + 2)
@@ -214,6 +225,7 @@ class MarshalItemExporterTest(BaseItemExporterTest):
 
 class CsvItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
+        self.output = tempfile.TemporaryFile()
         return CsvItemExporter(self.output, **kwargs)
 
     def assertCsvEqual(self, first, second, msg=None):
@@ -224,7 +236,8 @@ class CsvItemExporterTest(BaseItemExporterTest):
         return self.assertEqual(csvsplit(first), csvsplit(second), msg)
 
     def _check_output(self):
-        self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
+        self.output.seek(0)
+        self.assertCsvEqual(to_unicode(self.output.read()), u'age,name\r\n22,John\xa3\r\n')
 
     def assertExportResult(self, item, expected, **kwargs):
         fp = BytesIO()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index eef0384cf..4f72c0ff4 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from collections import OrderedDict
 import os
 import csv
 import json
@@ -578,6 +579,62 @@ class FeedExportTest(unittest.TestCase):
         yield self.assertExported(items, header, rows,
                                   settings=settings, ordered=True)
 
+        # fields may be defined as a comma-separated list
+        header = ["foo", "baz", "hello"]
+        settings = {'FEED_EXPORT_FIELDS': ",".join(header)}
+        rows = [
+            {'foo': 'bar1', 'baz': '',      'hello': ''},
+            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
+            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
+            {'foo': '',     'baz': '',      'hello': 'world4'},
+        ]
+        yield self.assertExported(items, header, rows,
+                                  settings=settings, ordered=True)
+
+        # fields may also be defined as a JSON array
+        header = ["foo", "baz", "hello"]
+        settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+        rows = [
+            {'foo': 'bar1', 'baz': '',      'hello': ''},
+            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
+            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
+            {'foo': '',     'baz': '',      'hello': 'world4'},
+        ]
+        yield self.assertExported(items, header, rows,
+                                  settings=settings, ordered=True)
+
+        # custom output field names can be specified
+        header = OrderedDict((
+            ("foo", "Foo"),
+            ("baz", "Baz"),
+            ("hello", "Hello"),
+        ))
+        settings = {'FEED_EXPORT_FIELDS': header}
+        rows = [
+            {'Foo': 'bar1', 'Baz': '',      'Hello': ''},
+            {'Foo': 'bar2', 'Baz': '',      'Hello': 'world2'},
+            {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},
+            {'Foo': '',     'Baz': '',      'Hello': 'world4'},
+        ]
+        yield self.assertExported(items, list(header.values()), rows,
+                                  settings=settings, ordered=True)
+
+        # custom output field names can be specified as a JSON object
+        header = OrderedDict((
+            ("foo", "Foo"),
+            ("baz", "Baz"),
+            ("hello", "Hello"),
+        ))
+        settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+        rows = [
+            {'Foo': 'bar1', 'Baz': '',      'Hello': ''},
+            {'Foo': 'bar2', 'Baz': '',      'Hello': 'world2'},
+            {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},
+            {'Foo': '',     'Baz': '',      'Hello': 'world4'},
+        ]
+        yield self.assertExported(items, list(header.values()), rows,
+                                  settings=settings, ordered=True)
+
     @defer.inlineCallbacks
     def test_export_dicts(self):
         # When dicts are used, only keys from the first row are used as

From 20719bac5cdc4898e9f01fcf4f92aa781a9d0ad1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 17 Dec 2019 15:09:43 +0100
Subject: [PATCH 033/716] Fix import error

---
 scrapy/settings/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index c1fff4d95..98421be18 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -224,7 +224,7 @@ class BaseSettings(MutableMapping):
         value = self.get(name, default)
         if value is None:
             return {}
-        if isinstance(value, six.string_types):
+        if isinstance(value, str):
             try:
                 return json.loads(value, object_pairs_hook=OrderedDict)
             except ValueError:

From 6d6243afbb16cb5b7d401a0b2ea7a174b7be71b8 Mon Sep 17 00:00:00 2001
From: leobalestri <33645316+leobalestri@users.noreply.github.com>
Date: Sun, 16 Feb 2020 23:45:41 -0800
Subject: [PATCH 034/716] Update install.rst

Minor grammar and typo fixes
---
 docs/intro/install.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 51b41b4d7..a08dedbd0 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -186,7 +186,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to
 successfully install Scrapy and its dependencies. Here are some proposed
 solutions:
 
-* *(Recommended)* **Don't** use system python, install a new, updated version
+* *(Recommended)* **Don't** use system python. Install a new, updated version
   that doesn't conflict with the rest of your system. Here's how to do it using
   the `homebrew`_ package manager:
 
@@ -231,9 +231,9 @@ PyPy
 We recommend using the latest PyPy version. The version tested is 5.9.0.
 For PyPy3, only Linux installation was tested.
 
-Most scrapy dependencides now have binary wheels for CPython, but not for PyPy.
-This means that these dependecies will be built during installation.
-On OS X, you are likely to face an issue with building Cryptography dependency,
+Most scrapy dependencies now have binary wheels for CPython, but not for PyPy.
+This means that these dependencies will be built during installation.
+On OS X, you are likely to face an issue with building Cryptography dependency. The
 solution to this problem is described
 `here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
 that is to ``brew install openssl`` and then export the flags that this command

From 5834088e670d93b2a63ad8afb258e687af0a9b88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 18 Feb 2020 14:18:15 +0100
Subject: [PATCH 035/716] Apply feedback

---
 scrapy/settings/__init__.py |   5 +-
 tests/test_feedexport.py    | 109 ++++++++++++++++++------------------
 2 files changed, 56 insertions(+), 58 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 98421be18..6f5b1ef97 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -207,8 +207,7 @@ class BaseSettings(MutableMapping):
         If it is a string it will be evaluated as JSON, or as a comma-separated
         list of strings as a fallback.
 
-        For example, settings populated through environment variables will
-        return:
+        For example, settings populated from the command line will return:
 
         -   ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to
             ``'{"key1": "value1", "key2": "value2"}'``
@@ -223,7 +222,7 @@ class BaseSettings(MutableMapping):
         """
         value = self.get(name, default)
         if value is None:
-            return {}
+            return OrderedDict()
         if isinstance(value, str):
             try:
                 return json.loads(value, object_pairs_hook=OrderedDict)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 291c47702..781cdc543 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -5,6 +5,7 @@ import warnings
 import tempfile
 import shutil
 import string
+import sys
 from collections import OrderedDict
 from io import BytesIO
 from pathlib import Path
@@ -12,6 +13,7 @@ from unittest import mock
 from urllib.parse import urljoin, urlparse, quote
 from urllib.request import pathname2url
 
+import pytest
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -590,78 +592,75 @@ class FeedExportTest(unittest.TestCase):
         yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
         yield self.assertExportedJsonLines(items, rows_jl)
 
-        # edge case: FEED_EXPORT_FIELDS==[] means the same as default None
+    @defer.inlineCallbacks
+    def test_export_items_empty_field_list(self):
+        # FEED_EXPORT_FIELDS==[] means the same as default None
+        items = [{'foo': 'bar'}]
+        header = ["foo"]
+        rows = [{'foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': []}
-        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
-        yield self.assertExportedJsonLines(items, rows_jl, settings)
+        yield self.assertExportedCsv(items, header, rows, ordered=False)
+        yield self.assertExportedJsonLines(items, rows, settings)
 
-        # it is possible to override fields using FEED_EXPORT_FIELDS
-        header = ["foo", "baz", "hello"]
+    @defer.inlineCallbacks
+    def test_export_items_field_list(self):
+        items = [{'foo': 'bar'}]
+        header = ["foo", "baz"]
+        rows = [{'foo': 'bar', 'baz': ''}]
         settings = {'FEED_EXPORT_FIELDS': header}
-        rows = [
-            {'foo': 'bar1', 'baz': '',      'hello': ''},
-            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
-            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
-            {'foo': '',     'baz': '',      'hello': 'world4'},
-        ]
-        yield self.assertExported(items, header, rows,
-                                  settings=settings, ordered=True)
+        yield self.assertExported(items, header, rows, settings=settings)
 
-        # fields may be defined as a comma-separated list
-        header = ["foo", "baz", "hello"]
+    @defer.inlineCallbacks
+    def test_export_items_comma_separated_field_list(self):
+        items = [{'foo': 'bar'}]
+        header = ["foo", "baz"]
+        rows = [{'foo': 'bar', 'baz': ''}]
         settings = {'FEED_EXPORT_FIELDS': ",".join(header)}
-        rows = [
-            {'foo': 'bar1', 'baz': '',      'hello': ''},
-            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
-            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
-            {'foo': '',     'baz': '',      'hello': 'world4'},
-        ]
-        yield self.assertExported(items, header, rows,
-                                  settings=settings, ordered=True)
+        yield self.assertExported(items, header, rows, settings=settings)
 
-        # fields may also be defined as a JSON array
-        header = ["foo", "baz", "hello"]
+    @defer.inlineCallbacks
+    def test_export_items_json_field_list(self):
+        items = [{'foo': 'bar'}]
+        header = ["foo", "baz"]
+        rows = [{'foo': 'bar', 'baz': ''}]
         settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
-        rows = [
-            {'foo': 'bar1', 'baz': '',      'hello': ''},
-            {'foo': 'bar2', 'baz': '',      'hello': 'world2'},
-            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
-            {'foo': '',     'baz': '',      'hello': 'world4'},
-        ]
-        yield self.assertExported(items, header, rows,
-                                  settings=settings, ordered=True)
+        yield self.assertExported(items, header, rows, settings=settings)
 
-        # custom output field names can be specified
+    @defer.inlineCallbacks
+    def test_export_items_field_names(self):
+        items = [{'foo': 'bar'}]
         header = OrderedDict((
             ("foo", "Foo"),
-            ("baz", "Baz"),
-            ("hello", "Hello"),
         ))
+        rows = [{'Foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': header}
-        rows = [
-            {'Foo': 'bar1', 'Baz': '',      'Hello': ''},
-            {'Foo': 'bar2', 'Baz': '',      'Hello': 'world2'},
-            {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},
-            {'Foo': '',     'Baz': '',      'Hello': 'world4'},
-        ]
         yield self.assertExported(items, list(header.values()), rows,
-                                  settings=settings, ordered=True)
+                                  settings=settings)
 
-        # custom output field names can be specified as a JSON object
+    @pytest.mark.skipif(sys.version_info < (3, 7),
+                        reason='Only official in Python 3.7+')
+    @defer.inlineCallbacks
+    def test_export_items_dict_field_names(self):
+        items = [{'foo': 'bar'}]
+        header = {
+            'baz': 'Baz',
+            'foo': 'Foo',
+        }
+        rows = [{'Baz': '', 'Foo': 'bar'}]
+        settings = {'FEED_EXPORT_FIELDS': header}
+        yield self.assertExported(items, ['Baz', 'Foo'], rows,
+                                  settings=settings)
+
+    @defer.inlineCallbacks
+    def test_export_items_json_field_names(self):
+        items = [{'foo': 'bar'}]
         header = OrderedDict((
             ("foo", "Foo"),
-            ("baz", "Baz"),
-            ("hello", "Hello"),
         ))
+        rows = [{'Foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
-        rows = [
-            {'Foo': 'bar1', 'Baz': '',      'Hello': ''},
-            {'Foo': 'bar2', 'Baz': '',      'Hello': 'world2'},
-            {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},
-            {'Foo': '',     'Baz': '',      'Hello': 'world4'},
-        ]
         yield self.assertExported(items, list(header.values()), rows,
-                                  settings=settings, ordered=True)
+                                  settings=settings)
 
     @defer.inlineCallbacks
     def test_export_dicts(self):
@@ -697,7 +696,7 @@ class FeedExportTest(unittest.TestCase):
                 {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
             ]
             yield self.assertExported(items, ['foo', 'baz', 'egg'], rows,
-                                      settings=settings, ordered=True)
+                                      settings=settings)
 
             # export a subset of columns
             settings = {'FEED_EXPORT_FIELDS': 'egg,baz'}
@@ -706,7 +705,7 @@ class FeedExportTest(unittest.TestCase):
                 {'egg': 'spam2', 'baz': 'quux2'}
             ]
             yield self.assertExported(items, ['egg', 'baz'], rows,
-                                      settings=settings, ordered=True)
+                                      settings=settings)
 
     @defer.inlineCallbacks
     def test_export_encoding(self):

From 231c9ddef8be9d749dcc2684f07ea9bb8bd02aa3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 28 Feb 2020 18:50:45 +0100
Subject: [PATCH 036/716] Update docs/intro/install.rst

---
 docs/intro/install.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index a08dedbd0..b71379e4d 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -186,7 +186,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to
 successfully install Scrapy and its dependencies. Here are some proposed
 solutions:
 
-* *(Recommended)* **Don't** use system python. Install a new, updated version
+* *(Recommended)* **Don't** use system Python. Install a new, updated version
   that doesn't conflict with the rest of your system. Here's how to do it using
   the `homebrew`_ package manager:
 

From 1d77eac950966463faee411b6196f197a8d32d4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Apr 2020 14:57:55 +0200
Subject: [PATCH 037/716] Fix Flake8-reported issues

---
 scrapy/http/request/form.py | 2 +-
 tests/test_http_request.py  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index d58d217b6..bdb6bec7a 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -33,7 +33,7 @@ class FormRequest(Request):
                 self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded')
                 self._set_body(form_query_str)
             else:
-                self._set_url(urlunsplit(urlsplit(self.url)._replace(query = form_query_str)))
+                self._set_url(urlunsplit(urlsplit(self.url)._replace(query=form_query_str)))
 
     @classmethod
     def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index bc34bb26d..96954d419 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -369,7 +369,7 @@ class FormRequestTest(RequestTest):
     def test_empty_formdata(self):
         r1 = self.request_class("http://www.example.com", formdata={})
         self.assertEqual(r1.body, b'')
-        
+
     def test_formdata_overrides_querystring(self):
         data = (('a', 'one'), ('a', 'two'), ('b', '2'))
         url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
@@ -378,7 +378,7 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs[b'b'], [b'2'])
         self.assertIsNone(fs.get(b'c'))
 
-        data = {'a' : '1', 'b' : '2'}
+        data = {'a': '1', 'b': '2'}
         fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data))
         self.assertEqual(fs[b'a'], [b'1'])
         self.assertEqual(fs[b'b'], [b'2'])

From 4605c66a80dabd64924e397580224a667cd73ec8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 7 May 2020 12:38:51 +0200
Subject: [PATCH 038/716] Fix AttributeError

---
 scrapy/utils/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 376c1f992..6a6d38a5c 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -114,7 +114,7 @@ def get_sources(use_closest=True):
 def feed_complete_default_values_from_settings(feed, settings):
     out = feed.copy()
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
-    out.setdefault("fields", settings.settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
+    out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
     if settings["FEED_EXPORT_INDENT"] is None:

From 9408c77a1e16df89feeab055d3530ebad66555e1 Mon Sep 17 00:00:00 2001
From: Aditya Kumar <k.aditya00@gmail.com>
Date: Sun, 31 May 2020 13:09:56 +0530
Subject: [PATCH 039/716] feat(http2): IH2EventsHandler, http2 module

---
 scrapy/core/http2/__init__.py |   0
 scrapy/core/http2/protocol.py | 136 ++++++++++++++++++++++++++++++++++
 2 files changed, 136 insertions(+)
 create mode 100644 scrapy/core/http2/__init__.py
 create mode 100644 scrapy/core/http2/protocol.py

diff --git a/scrapy/core/http2/__init__.py b/scrapy/core/http2/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
new file mode 100644
index 000000000..4a465168f
--- /dev/null
+++ b/scrapy/core/http2/protocol.py
@@ -0,0 +1,136 @@
+from h2.connection import H2Connection
+from h2.config import H2Configuration
+from h2.events import (
+    ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded,
+    StreamReset, TrailersReceived, WindowUpdated
+)
+
+from scrapy.http import Request
+
+from twisted.internet.defer import maybeDeferred
+from twisted.internet.protocol import Protocol
+
+from urllib.parse import urlparse
+
+from zope.interface import implementer, Interface
+
+
+class IH2EventsHandler(Interface):
+    def connection_terminated(event: ConnectionTerminated):
+        pass
+
+    def data_received(event: DataReceived):
+        pass
+
+    def response_received(event: ResponseReceived):
+        pass
+
+    def stream_ended(event: StreamEnded):
+        pass
+
+    def stream_reset(event: StreamReset):
+        pass
+
+    def trailers_received(event: TrailersReceived):
+        pass
+
+    def window_updated(event: WindowUpdated):
+        pass
+
+
+@implementer(IH2EventsHandler)
+class H2ClientProtocol(Protocol):
+    def __init__(self):
+        config = H2Configuration(client_side=True)
+        self.conn = H2Connection(config=config)
+        
+        # List of ongoing stream id's
+        self.streams = []
+
+    def request(self, _request: Request):
+        url = urlparse(_request.url)
+
+        request_headers = [
+            (':method', _request.method),
+            (':authority', url.netloc),
+            (':scheme', url.scheme),
+            (':path', url.path),
+        ]
+
+        # TODO: Check for user-agent while testing
+        request_headers += list(_request.headers.items())
+
+        # TODO: Add support for cookies here
+
+
+
+
+
+
+    def connectionMade(self):
+        """Called by Twisted when the connection is established. We can start
+        sending some data now: we should open with the connection preamble.
+        """
+        self.conn.initiate_connection()
+        self.transport.write(self.conn.data_to_send())
+
+    def dataReceived(self, data):
+        events = self.conn.receive_data(data)
+
+        self._handle_events(events)
+
+        _data = self.conn.data_to_send()
+        if _data:
+            self.transport.write(data)
+
+    def connectionLost(self, reason):
+        """Called by Twisted when the transport connection is lost.
+        """  
+
+        for stream_id in self.streams:
+            self.conn.end_stream(stream_id)
+
+    def _handle_events(self, events):
+        """Private method which acts as a bridge between the events
+        received from the HTTP/2 data and IH2EventsHandler
+
+        Arguments:
+            events {list} -- A list of events that the remote peer
+                triggered by sending data
+        """
+        for event in events:
+            if isinstance(event, ConnectionTerminated):
+                self.connection_terminated(event)
+            elif isinstance(event, DataReceived):
+                self.data_received(event)
+            elif isinstance(event, ResponseReceived):
+                self.response_received(event)
+            elif isinstance(event, StreamEnded):
+                self.stream_ended(event)
+            elif isinstance(event, StreamReset):
+                self.stream_reset(event)
+            elif isinstance(event, TrailersReceived):
+                self.trailers_received(event)
+            elif isinstance(event, WindowUpdated):
+                self.window_updated(event)
+
+    def connection_terminated(self, event):
+        pass
+
+    def data_received(self, event):
+        pass
+
+    def response_received(self, event):
+        pass
+
+    def stream_ended(self, event):
+        pass
+
+    def stream_reset(self, event):
+        pass
+
+    def trailers_received(self, event):
+        pass
+
+    def window_updated(self, event):
+        pass

From 791292334e86723a80cfd94b2877a258721f2c93 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Tue, 2 Jun 2020 09:13:31 +0530
Subject: [PATCH 040/716] chore(http2): Stream class

---
 scrapy/core/http2/protocol.py | 75 +++++++++++++++++++++++------------
 scrapy/core/http2/stream.py   | 44 ++++++++++++++++++++
 2 files changed, 94 insertions(+), 25 deletions(-)
 create mode 100644 scrapy/core/http2/stream.py

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 4a465168f..b95cb05ae 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -6,8 +6,9 @@ from h2.events import (
 )
 
 from scrapy.http import Request
+from scrapy.core.http2.stream import Stream
 
-from twisted.internet.defer import maybeDeferred
+from twisted.internet.defer import Deferred
 from twisted.internet.protocol import Protocol
 
 from urllib.parse import urlparse
@@ -40,31 +41,55 @@ class IH2EventsHandler(Interface):
 
 @implementer(IH2EventsHandler)
 class H2ClientProtocol(Protocol):
+    # TODO: Check for user-agent while testing
+    # TODO: Add support for cookies
+    # TODO: Handle priority updates
+
     def __init__(self):
         config = H2Configuration(client_side=True)
         self.conn = H2Connection(config=config)
-        
-        # List of ongoing stream id's
-        self.streams = []
+
+        # ID of the next request stream
+        # Assuming each request stream creates a new response stream
+        # we increment by 2 for each new request stream created
+        self.next_stream_id = 1
+
+        # Streams are stored in a dictionary keyed off their stream IDs
+        self.streams = {}
+
+    def _new_stream(self, headers):
+        """Instantiates a new Stream object
+        """
+        stream = Stream(self.next_stream_id, headers)
+
+        self.next_stream_id += 2
+
+        return stream
 
     def request(self, _request: Request):
+        """
+
+        Arguments:
+            _request {Request} -- [description]
+        """
         url = urlparse(_request.url)
 
-        request_headers = [
-            (':method', _request.method),
-            (':authority', url.netloc),
-            (':scheme', url.scheme),
-            (':path', url.path),
-        ]
-
-        # TODO: Check for user-agent while testing
-        request_headers += list(_request.headers.items())
-
-        # TODO: Add support for cookies here
+        _request[":method"] = _request.method
 
+        # TODO: Make authority private class variable instead
+        # of parsing it from request url all requests to same
+        # host are multiplexed into one connection & a connection
+        # can have only 1 host at a time
+        _request[":authority"] = url.netloc
 
+        # TODO: Check if scheme can be 'http' for HTTP/2 ?
+        _request[":scheme"] = "https"
+        _request[":path"] = url.path
 
+        stream = self._new_stream(_request.headers)
+        d = stream.get_response()
 
+        return d
 
 
     def connectionMade(self):
@@ -76,7 +101,6 @@ class H2ClientProtocol(Protocol):
 
     def dataReceived(self, data):
         events = self.conn.receive_data(data)
-
         self._handle_events(events)
 
         _data = self.conn.data_to_send()
@@ -85,9 +109,10 @@ class H2ClientProtocol(Protocol):
 
     def connectionLost(self, reason):
         """Called by Twisted when the transport connection is lost.
-        """  
+        """
 
-        for stream_id in self.streams:
+        for stream_id in self.streams.keys():
+            # TODO: Close each Stream instance in a clean manner
             self.conn.end_stream(stream_id)
 
     def _handle_events(self, events):
@@ -114,23 +139,23 @@ class H2ClientProtocol(Protocol):
             elif isinstance(event, WindowUpdated):
                 self.window_updated(event)
 
-    def connection_terminated(self, event):
+    def connection_terminated(self, event: ConnectionTerminated):
         pass
 
-    def data_received(self, event):
+    def data_received(self, event: DataReceived):
         pass
 
-    def response_received(self, event):
+    def response_received(self, event: ResponseReceived):
         pass
 
-    def stream_ended(self, event):
+    def stream_ended(self, event: StreamEnded):
         pass
 
-    def stream_reset(self, event):
+    def stream_reset(self, event: StreamReset):
         pass
 
-    def trailers_received(self, event):
+    def trailers_received(self, event: TrailersReceived):
         pass
 
-    def window_updated(self, event):
+    def window_updated(self, event: WindowUpdated):
         pass
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
new file mode 100644
index 000000000..558a6552c
--- /dev/null
+++ b/scrapy/core/http2/stream.py
@@ -0,0 +1,44 @@
+from scrapy.http.headers import Headers
+
+
+class Stream:
+    """Represents a single HTTP/2 Stream.
+
+    Stream is a bidirectional flow of bytes within an established connection,
+    which may carry one or more messages. Handles the tranfer of HTTP Headers
+    and Data frames.
+    """
+
+    def __init__(self, stream_id, headers):
+        """
+        Arguments:
+            stream_id {int} -- For one HTTP/2 connection each stream is
+                uniquely identified by a single integer
+            headers {Headers} -- HTTP request headers
+        """
+
+        # Headers received after sending the request
+        self.response_headers = Headers({})
+
+        # Headers which are send with the request
+        # These cannot be modified any furthur
+        self._request_headers = headers
+
+        # TODO: Add canceller for the Deferred below
+        self._deferred_response = Deferred()
+
+    def get_response(self):
+        """Simply return a Deferred which fires when response
+        from the asynchronous request is available
+
+        Returns:
+            Deferred -- Calls the callback when the response is
+                avaialble
+        """
+        return self._deferred_response
+
+    def receive_data(self, data):
+        pass
+
+    def receive_headers(self, headers):
+        pass

From bdabc500aaa37026dfceddf94b41e80f9ce2ce6b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 6 Jun 2020 16:32:43 -0300
Subject: [PATCH 041/716] Update headless browser docs

---
 docs/topics/dynamic-content.rst | 36 ++++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 495111b56..7450de4a2 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -248,8 +248,34 @@ Using a headless browser
 A `headless browser`_ is a special web browser that provides an API for
 automation.
 
-The easiest way to use a headless browser with Scrapy is to use Selenium_,
-along with `scrapy-selenium`_ for seamless integration.
+Since version 2.0, it is possible to integrate libraries that use the
+``async/await`` syntax. One such library is `pyppeteer`_ (an unnoficial
+Python port of `puppeteer`_), which uses headless Chrome to download and
+render pages.
+The following is a simple snippet to illustrate its usage within Scrapy::
+
+    import pyppeteer
+    import scrapy
+
+    class PyppeteerSpider(scrapy.Spider):
+        name = "pyppeteer"
+        start_urls = ["data:,"]  # avoid making an actual upstream request
+
+        async def parse(self, response):
+            browser = await pyppeteer.launch()
+            page = await browser.newPage()
+            await page.goto("https:/example.org")
+            title = await page.title()
+            yield {"title": title}
+
+Keep in mind that this is just a proof of concept, since it circumvents
+most of the Scrapy components (middlewares, dupefilter, etc).
+
+There are some 3rd party projects which provider better integration:
+
+* https://github.com/elacuesta/scrapy-pyppeteer
+* https://github.com/lopuhin/scrapy-pyppeteer
+* https://github.com/clemfromspace/scrapy-puppeteer
 
 
 .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
@@ -259,11 +285,11 @@ along with `scrapy-selenium`_ for seamless integration.
 .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
 .. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
 .. _js2xml: https://github.com/scrapinghub/js2xml
+.. _puppeteer: https://pptr.dev/
+.. _pyppeteer: https://pyppeteer.github.io/pyppeteer/
 .. _pytesseract: https://github.com/madmaze/pytesseract
-.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
 .. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
-.. _Selenium: https://www.selenium.dev/
 .. _Splash: https://github.com/scrapinghub/splash
 .. _tabula-py: https://github.com/chezou/tabula-py
 .. _wget: https://www.gnu.org/software/wget/
-.. _wgrep: https://github.com/stav/wgrep
\ No newline at end of file
+.. _wgrep: https://github.com/stav/wgrep

From 9ff9caecadf8215ce75b0ad4231b8289bca168fe Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sun, 7 Jun 2020 14:04:53 +0530
Subject: [PATCH 042/716] feat(http2): support for GET requests

---
 scrapy/core/http2/protocol.py | 126 ++++++++++++++++++++++------------
 scrapy/core/http2/stream.py   |  71 +++++++++++++++----
 2 files changed, 139 insertions(+), 58 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index b95cb05ae..847e74f97 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -1,20 +1,19 @@
-from h2.connection import H2Connection
+import logging
+
 from h2.config import H2Configuration
+from h2.connection import H2Connection
 from h2.events import (
-    ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded,
-    StreamReset, TrailersReceived, WindowUpdated
+    ConnectionTerminated, DataReceived, ResponseReceived, RemoteSettingsChanged,
+    StreamEnded, StreamReset, TrailersReceived, WindowUpdated
 )
-
-from scrapy.http import Request
-from scrapy.core.http2.stream import Stream
-
-from twisted.internet.defer import Deferred
-from twisted.internet.protocol import Protocol
-
-from urllib.parse import urlparse
-
+from twisted.internet.protocol import connectionDone, Protocol
 from zope.interface import implementer, Interface
 
+from scrapy.core.http2.stream import Stream
+from scrapy.http import Request
+
+LOGGER = logging.getLogger(__name__)
+
 
 class IH2EventsHandler(Interface):
     def connection_terminated(event: ConnectionTerminated):
@@ -26,6 +25,9 @@ class IH2EventsHandler(Interface):
     def response_received(event: ResponseReceived):
         pass
 
+    def remote_settings_changed(event: RemoteSettingsChanged):
+        pass
+
     def stream_ended(event: StreamEnded):
         pass
 
@@ -46,7 +48,7 @@ class H2ClientProtocol(Protocol):
     # TODO: Handle priority updates
 
     def __init__(self):
-        config = H2Configuration(client_side=True)
+        config = H2Configuration(client_side=True, header_encoding='utf-8')
         self.conn = H2Connection(config=config)
 
         # ID of the next request stream
@@ -57,60 +59,69 @@ class H2ClientProtocol(Protocol):
         # Streams are stored in a dictionary keyed off their stream IDs
         self.streams = {}
 
-    def _new_stream(self, headers):
+        # Boolean to keep track the connection is made
+        # If requests are received before connection is made
+        # we keep all requests in a pool and send them as the connection
+        # is made
+        self.is_connection_made = False
+        self._pending_request_stream_pool = []
+
+    def _new_stream(self, request: Request):
         """Instantiates a new Stream object
         """
-        stream = Stream(self.next_stream_id, headers)
-
+        stream = Stream(self.next_stream_id, request, self)
         self.next_stream_id += 2
 
+        self.streams[stream.stream_id] = stream
         return stream
 
+    def _write_to_transport(self):
+        """ Write data to the underlying transport connection
+        from the HTTP2 connection instance if any
+        """
+        data = self.conn.data_to_send()
+        if data:
+            self.transport.write(data)
+
     def request(self, _request: Request):
-        """
-
-        Arguments:
-            _request {Request} -- [description]
-        """
-        url = urlparse(_request.url)
-
-        _request[":method"] = _request.method
-
-        # TODO: Make authority private class variable instead
-        # of parsing it from request url all requests to same
-        # host are multiplexed into one connection & a connection
-        # can have only 1 host at a time
-        _request[":authority"] = url.netloc
-
-        # TODO: Check if scheme can be 'http' for HTTP/2 ?
-        _request[":scheme"] = "https"
-        _request[":path"] = url.path
-
-        stream = self._new_stream(_request.headers)
+        stream = self._new_stream(_request)
         d = stream.get_response()
 
-        return d
+        # If connection is not yet established then add the
+        # stream to pool or initiate request
+        if self.is_connection_made:
+            stream.initiate_request()
+        else:
+            self._pending_request_stream_pool.append(stream)
 
+        return d
 
     def connectionMade(self):
         """Called by Twisted when the connection is established. We can start
         sending some data now: we should open with the connection preamble.
         """
+        LOGGER.info("Connection made to {}".format(self.transport))
         self.conn.initiate_connection()
-        self.transport.write(self.conn.data_to_send())
+        self._write_to_transport()
+
+        self.is_connection_made = True
+
+        # Initiate all pending requests
+        for stream in self._pending_request_stream_pool:
+            assert isinstance(stream, Stream)
+            stream.initiate_request()
+
+        self._pending_request_stream_pool.clear()
 
     def dataReceived(self, data):
         events = self.conn.receive_data(data)
         self._handle_events(events)
+        self._write_to_transport()
 
-        _data = self.conn.data_to_send()
-        if _data:
-            self.transport.write(data)
+    def connectionLost(self, reason=connectionDone):
 
-    def connectionLost(self, reason):
         """Called by Twisted when the transport connection is lost.
         """
-
         for stream_id in self.streams.keys():
             # TODO: Close each Stream instance in a clean manner
             self.conn.end_stream(stream_id)
@@ -138,18 +149,43 @@ class H2ClientProtocol(Protocol):
                 self.trailers_received(event)
             elif isinstance(event, WindowUpdated):
                 self.window_updated(event)
+            elif isinstance(event, RemoteSettingsChanged):
+                self.remote_settings_changed(event)
+
+    def send_headers(self, stream_id, headers):
+        """ Send the headers for a given stream to the resource
+        Initiates a new connection hence.
+
+        Arguments:
+            stream_id {int} -- Valid stream id
+            headers {List[Tuple[str, str]]} -- Headers of the request
+        """
+        if stream_id in self.streams:
+            self.conn.send_headers(stream_id, headers, end_stream=True)
+            self._write_to_transport()
+        else:
+            pass
 
     def connection_terminated(self, event: ConnectionTerminated):
         pass
 
     def data_received(self, event: DataReceived):
-        pass
+        stream_id = event.stream_id
+        # TODO: Stream do not exist in self.streams dict
+        self.streams[stream_id].receive_data(event.data)
 
     def response_received(self, event: ResponseReceived):
+        stream_id = event.stream_id
+        # TODO: Stream do not exist in self.streams dict
+        self.streams[stream_id].receive_headers(event.headers)
+
+    def remote_settings_changed(self, event: RemoteSettingsChanged):
         pass
 
     def stream_ended(self, event: StreamEnded):
-        pass
+        stream_id = event.stream_id
+        # TODO: Stream do not exist in self.streams dict
+        self.streams[stream_id].end_stream()
 
     def stream_reset(self, event: StreamReset):
         pass
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 558a6552c..d2a9f02fa 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -1,3 +1,8 @@
+from urllib.parse import urlparse
+
+from twisted.internet.defer import Deferred
+
+from scrapy.http import Request, Response
 from scrapy.http.headers import Headers
 
 
@@ -5,24 +10,30 @@ class Stream:
     """Represents a single HTTP/2 Stream.
 
     Stream is a bidirectional flow of bytes within an established connection,
-    which may carry one or more messages. Handles the tranfer of HTTP Headers
+    which may carry one or more messages. Handles the transfer of HTTP Headers
     and Data frames.
+
+    Role of this class is to
+    1. Combine all the data frames
     """
 
-    def __init__(self, stream_id, headers):
+    def __init__(self, stream_id: int, request: Request, connection):
         """
         Arguments:
             stream_id {int} -- For one HTTP/2 connection each stream is
                 uniquely identified by a single integer
-            headers {Headers} -- HTTP request headers
+            request {Request} -- HTTP request
+            connection {H2ClientProtocol} -- HTTP/2 connection this stream belongs to
         """
 
-        # Headers received after sending the request
-        self.response_headers = Headers({})
+        self.stream_id = stream_id
+        self._request = request
+        self._conn = connection
 
-        # Headers which are send with the request
-        # These cannot be modified any furthur
-        self._request_headers = headers
+        self._response_data = b""
+
+        # Headers received after sending the request
+        self._response_headers = Headers({})
 
         # TODO: Add canceller for the Deferred below
         self._deferred_response = Deferred()
@@ -32,13 +43,47 @@ class Stream:
         from the asynchronous request is available
 
         Returns:
-            Deferred -- Calls the callback when the response is
-                avaialble
+            Deferred -- Calls the callback passing the response
         """
         return self._deferred_response
 
-    def receive_data(self, data):
-        pass
+    def initiate_request(self):
+        http2_request_headers = []
+        for name, value in self._request.headers.items():
+            http2_request_headers.append((name, value))
+
+        url = urlparse(self._request.url)
+        http2_request_headers += [
+            (":method", self._request.method),
+            (":authority", url.netloc),
+
+            # TODO: Check if scheme can be "http" for HTTP/2 ?
+            (":scheme", "https"),
+            (":path", url.path)
+        ]
+
+        self._conn.send_headers(self.stream_id, http2_request_headers)
+
+    def receive_data(self, data: bytes):
+        self._response_data += data
 
     def receive_headers(self, headers):
-        pass
+        for name, value in headers:
+            self._response_headers[name] = value
+
+    def end_stream(self):
+        """Stream is ended by the resource hence no further
+        data or headers should be expected on this stream.
+
+        We will call the response deferred callback passing
+        the response object
+        """
+        # TODO: Set flags, certificate, ip_address
+        response = Response(
+            url=self._request.url,
+            status=self._response_headers[":status"],
+            headers=self._response_headers,
+            body=self._response_data,
+            request=self._request
+        )
+        self._deferred_response.callback(response)

From 78aa1b2bfc3eceb09f2fa3dea59811375d6ed1f8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Mon, 8 Jun 2020 11:19:15 -0300
Subject: [PATCH 043/716] Fix typo

---
 docs/topics/dynamic-content.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 7450de4a2..e244eb7ff 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -271,7 +271,7 @@ The following is a simple snippet to illustrate its usage within Scrapy::
 Keep in mind that this is just a proof of concept, since it circumvents
 most of the Scrapy components (middlewares, dupefilter, etc).
 
-There are some 3rd party projects which provider better integration:
+There are some 3rd party projects which provide better integration:
 
 * https://github.com/elacuesta/scrapy-pyppeteer
 * https://github.com/lopuhin/scrapy-pyppeteer

From b6c5289fb900beb552a1ab608f572d99c180b4fb Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 10 Jun 2020 12:11:49 -0300
Subject: [PATCH 044/716] Close page in pyppeteer example, mention asyncio
 reactor

---
 docs/topics/dynamic-content.rst | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index e244eb7ff..56c8b6ae9 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -266,12 +266,16 @@ The following is a simple snippet to illustrate its usage within Scrapy::
             page = await browser.newPage()
             await page.goto("https:/example.org")
             title = await page.title()
+            await page.close()
             yield {"title": title}
 
+For this example to work, Scrapy needs to be running on top of the
+:ref:`asyncio reactor <install-asyncio>`.
+
 Keep in mind that this is just a proof of concept, since it circumvents
 most of the Scrapy components (middlewares, dupefilter, etc).
 
-There are some 3rd party projects which provide better integration:
+The following is a list of 3rd party projects which provide better integration:
 
 * https://github.com/elacuesta/scrapy-pyppeteer
 * https://github.com/lopuhin/scrapy-pyppeteer

From d09ccf8d3b2932d9393e6ce4a20c46befdd43acf Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sat, 13 Jun 2020 20:40:01 +0530
Subject: [PATCH 045/716] feat(http2): support for POST requests

BREAKING CHANGES
- Request is sent successfully with its Response received as well.
However, the StreamEnded event is not received which do not fires the
response deferred
---
 scrapy/core/http2/protocol.py | 108 +++++++++++++++++-----------------
 scrapy/core/http2/stream.py   |  79 +++++++++++++++++++++++--
 2 files changed, 127 insertions(+), 60 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 847e74f97..6d926b100 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -1,4 +1,5 @@
 import logging
+from typing import Dict, List
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
@@ -7,45 +8,19 @@ from h2.events import (
     StreamEnded, StreamReset, TrailersReceived, WindowUpdated
 )
 from twisted.internet.protocol import connectionDone, Protocol
-from zope.interface import implementer, Interface
 
 from scrapy.core.http2.stream import Stream
 from scrapy.http import Request
 
 LOGGER = logging.getLogger(__name__)
+LOGGER.debug = print
 
 
-class IH2EventsHandler(Interface):
-    def connection_terminated(event: ConnectionTerminated):
-        pass
-
-    def data_received(event: DataReceived):
-        pass
-
-    def response_received(event: ResponseReceived):
-        pass
-
-    def remote_settings_changed(event: RemoteSettingsChanged):
-        pass
-
-    def stream_ended(event: StreamEnded):
-        pass
-
-    def stream_reset(event: StreamReset):
-        pass
-
-    def trailers_received(event: TrailersReceived):
-        pass
-
-    def window_updated(event: WindowUpdated):
-        pass
-
-
-@implementer(IH2EventsHandler)
 class H2ClientProtocol(Protocol):
-    # TODO: Check for user-agent while testing
-    # TODO: Add support for cookies
-    # TODO: Handle priority updates
+    # TODO:
+    #  1. Check for user-agent while testing
+    #  2. Add support for cookies
+    #  3. Handle priority updates
 
     def __init__(self):
         config = H2Configuration(client_side=True, header_encoding='utf-8')
@@ -57,14 +32,14 @@ class H2ClientProtocol(Protocol):
         self.next_stream_id = 1
 
         # Streams are stored in a dictionary keyed off their stream IDs
-        self.streams = {}
+        self.streams: Dict[int, Stream] = {}
 
         # Boolean to keep track the connection is made
         # If requests are received before connection is made
         # we keep all requests in a pool and send them as the connection
         # is made
         self.is_connection_made = False
-        self._pending_request_stream_pool = []
+        self._pending_request_stream_pool: List[Stream] = []
 
     def _new_stream(self, request: Request):
         """Instantiates a new Stream object
@@ -100,29 +75,27 @@ class H2ClientProtocol(Protocol):
         """Called by Twisted when the connection is established. We can start
         sending some data now: we should open with the connection preamble.
         """
-        LOGGER.info("Connection made to {}".format(self.transport))
+        LOGGER.debug("Connection made to {}".format(self.transport))
         self.conn.initiate_connection()
         self._write_to_transport()
 
         self.is_connection_made = True
 
-        # Initiate all pending requests
-        for stream in self._pending_request_stream_pool:
-            assert isinstance(stream, Stream)
-            stream.initiate_request()
-
-        self._pending_request_stream_pool.clear()
-
     def dataReceived(self, data):
         events = self.conn.receive_data(data)
         self._handle_events(events)
         self._write_to_transport()
 
     def connectionLost(self, reason=connectionDone):
-
         """Called by Twisted when the transport connection is lost.
         """
-        for stream_id in self.streams.keys():
+        LOGGER.debug(f"connectionLost {reason}")
+        stream_ids = list(self.streams.keys())
+
+        for stream in self._pending_request_stream_pool:
+            stream_ids.remove(stream.stream_id)
+
+        for stream_id in stream_ids:
             # TODO: Close each Stream instance in a clean manner
             self.conn.end_stream(stream_id)
 
@@ -135,6 +108,7 @@ class H2ClientProtocol(Protocol):
                 triggered by sending data
         """
         for event in events:
+            LOGGER.debug(event)
             if isinstance(event, ConnectionTerminated):
                 self.connection_terminated(event)
             elif isinstance(event, DataReceived):
@@ -153,38 +127,62 @@ class H2ClientProtocol(Protocol):
                 self.remote_settings_changed(event)
 
     def send_headers(self, stream_id, headers):
-        """ Send the headers for a given stream to the resource
+        """Send the headers for a given stream to the resource
         Initiates a new connection hence.
+        This function is wrapper for :func:`~h2.connection.H2Connection.send_headers`
 
         Arguments:
             stream_id {int} -- Valid stream id
             headers {List[Tuple[str, str]]} -- Headers of the request
         """
-        if stream_id in self.streams:
-            self.conn.send_headers(stream_id, headers, end_stream=True)
-            self._write_to_transport()
-        else:
-            pass
+        LOGGER.debug(f'Send Headers: stream_id={stream_id} headers={headers}')
+        self.conn.send_headers(stream_id, headers, end_stream=False)
 
+    def send_data(self, stream_id, data):
+        """Send the data for a given stream to the resource.
+        Requires request headers to be sent at least once before this
+        function is called.
+        This function is wrapper for :func:`~h2.connection.H2Connection.send_data`
+
+        Arguments:
+            stream_id {int} -- Valid stream id
+            data {bytes} -- The data to send on the stream.
+        """
+        LOGGER.debug(f"Send Data: stream_id={stream_id} data={data}")
+        self.conn.send_data(stream_id, data, end_stream=False)
+
+    def end_stream(self, stream_id):
+        """End the given stream.
+        This function is wrapper for :func:`~h2.connection.H2Connection.end_stream`
+
+        Arguments:
+             stream_id {int} - Valid stream id
+        """
+        LOGGER.debug(f"End Stream: stream_id={stream_id}")
+        self.conn.end_stream(stream_id)
+
+    # Event handler functions starts here
     def connection_terminated(self, event: ConnectionTerminated):
         pass
 
     def data_received(self, event: DataReceived):
         stream_id = event.stream_id
-        # TODO: Stream do not exist in self.streams dict
         self.streams[stream_id].receive_data(event.data)
 
     def response_received(self, event: ResponseReceived):
         stream_id = event.stream_id
-        # TODO: Stream do not exist in self.streams dict
         self.streams[stream_id].receive_headers(event.headers)
 
     def remote_settings_changed(self, event: RemoteSettingsChanged):
-        pass
+        # TODO: handle MAX_CONCURRENT_STREAMS
+        # Initiate all pending requests
+        for stream in self._pending_request_stream_pool:
+            stream.initiate_request()
+
+        self._pending_request_stream_pool.clear()
 
     def stream_ended(self, event: StreamEnded):
         stream_id = event.stream_id
-        # TODO: Stream do not exist in self.streams dict
         self.streams[stream_id].end_stream()
 
     def stream_reset(self, event: StreamReset):
@@ -194,4 +192,6 @@ class H2ClientProtocol(Protocol):
         pass
 
     def window_updated(self, event: WindowUpdated):
-        pass
+        stream_id = event.stream_id
+        if stream_id != 0:
+            self.streams[stream_id].window_updated()
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index d2a9f02fa..ab5a0fc88 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -23,13 +23,23 @@ class Stream:
             stream_id {int} -- For one HTTP/2 connection each stream is
                 uniquely identified by a single integer
             request {Request} -- HTTP request
-            connection {H2ClientProtocol} -- HTTP/2 connection this stream belongs to
+            connection {H2Connection} -- HTTP/2 connection this stream belongs to.
         """
-
         self.stream_id = stream_id
         self._request = request
-        self._conn = connection
+        self._client_protocol = connection
 
+        self._request_body = self._request.body
+        self.content_length = 0 if self._request_body is None else len(self._request_body)
+
+        # Each time we send a data frame, we will decrease value by the amount send.
+        self.remaining_content_length = self.content_length
+
+        # Flag to keep track whether we have ended this stream
+        self.stream_ended = True
+
+        # Data received frame by frame from the server is appended
+        # and passed to the response Deferred when completely received.
         self._response_data = b""
 
         # Headers received after sending the request
@@ -59,10 +69,67 @@ class Stream:
 
             # TODO: Check if scheme can be "http" for HTTP/2 ?
             (":scheme", "https"),
-            (":path", url.path)
+            (":path", url.path),
+            # ("Content-Length", str(self.content_length))
+
+            # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers
+            #  are sent for request having body
         ]
 
-        self._conn.send_headers(self.stream_id, http2_request_headers)
+        self._client_protocol.send_headers(self.stream_id, http2_request_headers)
+        self.send_data()
+
+    def send_data(self):
+        """Called immediately after the headers are sent. Here we send all the
+         data as part of the request.
+
+         If the content length is 0 initially then we end the stream immediately and
+         wait for response data.
+         """
+
+        # TODO:
+        #  1. Add test for sending very large data
+        #  2. Add test for small data
+        #  3. Both (1) and (2) should be tested for
+        #    3.1 Large number of request
+        #    3.2 Small number of requests
+
+        # Firstly, check what the flow control window is for current stream.
+        window_size = self._client_protocol.conn.local_flow_control_window(stream_id=self.stream_id)
+
+        # Next, check what the maximum frame size is.
+        max_frame_size = self._client_protocol.conn.max_outbound_frame_size
+
+        # We will send no more than the window size or the remaining file size
+        # of data in this call, whichever is smaller.
+        bytes_to_send = min(window_size, self.remaining_content_length)
+
+        # We now need to send a number of data frames.
+        while bytes_to_send > 0:
+            chunk_size = min(bytes_to_send, max_frame_size)
+
+            data_chunk_start = self.content_length - self.remaining_content_length
+            data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size]
+
+            self._client_protocol.send_data(self.stream_id, data_chunk, end_stream=False)
+
+            bytes_to_send = max(0, bytes_to_send - chunk_size)
+            self.remaining_content_length = max(0, self.remaining_content_length - chunk_size)
+
+        # End the stream if no more data has to be send
+        if self.remaining_content_length == 0:
+            self._client_protocol.end_stream(self.stream_id)
+        else:
+            # TODO: Continue from here :)
+            pass
+
+    def window_updated(self):
+        """Flow control window size was changed.
+        Send data that earlier could not be sent as we were
+        blocked behind the flow control.
+        """
+        if self.remaining_content_length > 0 and not self.stream_ended:
+            self.send_data()
 
     def receive_data(self, data: bytes):
         self._response_data += data
@@ -72,7 +139,7 @@ class Stream:
             self._response_headers[name] = value
 
     def end_stream(self):
-        """Stream is ended by the resource hence no further
+        """Stream is ended by the server hence no further
         data or headers should be expected on this stream.
 
         We will call the response deferred callback passing

From d06bb12e351f61ee20c28626b003f4b59fd689f7 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sat, 13 Jun 2020 22:29:16 +0530
Subject: [PATCH 046/716] refactor: move H2Connection instance to stream
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Remove all wrapper funtions made such that stream can send header/data
to H2Connection as they were not necessary

BREAKING CHANGES
Looks like, for small set of response data the StreamEnded event is
emitted and everything works well -- tested for both GET & POST request.
Maybe some issue with window size and/or flow control as when the
response data needs to be broken into separate chunks -- not all chunks
are received everytime which leads to indefinite waiting for next data
chunk and the connection is lost due to timeout. 😥

Working on setting up testing environment now. After testing is setup
I'll debug the above bug furthur.
---
 scrapy/core/http2/protocol.py | 74 ++++++++---------------------------
 scrapy/core/http2/stream.py   | 16 ++++----
 2 files changed, 24 insertions(+), 66 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 6d926b100..4036dfb3e 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -4,8 +4,7 @@ from typing import Dict, List
 from h2.config import H2Configuration
 from h2.connection import H2Connection
 from h2.events import (
-    ConnectionTerminated, DataReceived, ResponseReceived, RemoteSettingsChanged,
-    StreamEnded, StreamReset, TrailersReceived, WindowUpdated
+    ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, StreamReset, WindowUpdated
 )
 from twisted.internet.protocol import connectionDone, Protocol
 
@@ -13,7 +12,6 @@ from scrapy.core.http2.stream import Stream
 from scrapy.http import Request
 
 LOGGER = logging.getLogger(__name__)
-LOGGER.debug = print
 
 
 class H2ClientProtocol(Protocol):
@@ -44,19 +42,27 @@ class H2ClientProtocol(Protocol):
     def _new_stream(self, request: Request):
         """Instantiates a new Stream object
         """
-        stream = Stream(self.next_stream_id, request, self)
+        stream = Stream(self.next_stream_id, request, self.conn)
         self.next_stream_id += 2
 
         self.streams[stream.stream_id] = stream
         return stream
 
+    def _send_pending_requests(self):
+        # TODO: handle MAX_CONCURRENT_STREAMS
+        # Initiate all pending requests
+        for stream in self._pending_request_stream_pool:
+            stream.initiate_request()
+            self._write_to_transport()
+
+        self._pending_request_stream_pool.clear()
+
     def _write_to_transport(self):
         """ Write data to the underlying transport connection
         from the HTTP2 connection instance if any
         """
         data = self.conn.data_to_send()
-        if data:
-            self.transport.write(data)
+        self.transport.write(data)
 
     def request(self, _request: Request):
         stream = self._new_stream(_request)
@@ -75,10 +81,11 @@ class H2ClientProtocol(Protocol):
         """Called by Twisted when the connection is established. We can start
         sending some data now: we should open with the connection preamble.
         """
-        LOGGER.debug("Connection made to {}".format(self.transport))
         self.conn.initiate_connection()
         self._write_to_transport()
 
+        self._send_pending_requests()
+
         self.is_connection_made = True
 
     def dataReceived(self, data):
@@ -89,7 +96,6 @@ class H2ClientProtocol(Protocol):
     def connectionLost(self, reason=connectionDone):
         """Called by Twisted when the transport connection is lost.
         """
-        LOGGER.debug(f"connectionLost {reason}")
         stream_ids = list(self.streams.keys())
 
         for stream in self._pending_request_stream_pool:
@@ -119,47 +125,10 @@ class H2ClientProtocol(Protocol):
                 self.stream_ended(event)
             elif isinstance(event, StreamReset):
                 self.stream_reset(event)
-            elif isinstance(event, TrailersReceived):
-                self.trailers_received(event)
             elif isinstance(event, WindowUpdated):
                 self.window_updated(event)
-            elif isinstance(event, RemoteSettingsChanged):
-                self.remote_settings_changed(event)
-
-    def send_headers(self, stream_id, headers):
-        """Send the headers for a given stream to the resource
-        Initiates a new connection hence.
-        This function is wrapper for :func:`~h2.connection.H2Connection.send_headers`
-
-        Arguments:
-            stream_id {int} -- Valid stream id
-            headers {List[Tuple[str, str]]} -- Headers of the request
-        """
-        LOGGER.debug(f'Send Headers: stream_id={stream_id} headers={headers}')
-        self.conn.send_headers(stream_id, headers, end_stream=False)
-
-    def send_data(self, stream_id, data):
-        """Send the data for a given stream to the resource.
-        Requires request headers to be sent at least once before this
-        function is called.
-        This function is wrapper for :func:`~h2.connection.H2Connection.send_data`
-
-        Arguments:
-            stream_id {int} -- Valid stream id
-            data {bytes} -- The data to send on the stream.
-        """
-        LOGGER.debug(f"Send Data: stream_id={stream_id} data={data}")
-        self.conn.send_data(stream_id, data, end_stream=False)
-
-    def end_stream(self, stream_id):
-        """End the given stream.
-        This function is wrapper for :func:`~h2.connection.H2Connection.end_stream`
-
-        Arguments:
-             stream_id {int} - Valid stream id
-        """
-        LOGGER.debug(f"End Stream: stream_id={stream_id}")
-        self.conn.end_stream(stream_id)
+            else:
+                LOGGER.info("Received unhandled event {}".format(event))
 
     # Event handler functions starts here
     def connection_terminated(self, event: ConnectionTerminated):
@@ -173,14 +142,6 @@ class H2ClientProtocol(Protocol):
         stream_id = event.stream_id
         self.streams[stream_id].receive_headers(event.headers)
 
-    def remote_settings_changed(self, event: RemoteSettingsChanged):
-        # TODO: handle MAX_CONCURRENT_STREAMS
-        # Initiate all pending requests
-        for stream in self._pending_request_stream_pool:
-            stream.initiate_request()
-
-        self._pending_request_stream_pool.clear()
-
     def stream_ended(self, event: StreamEnded):
         stream_id = event.stream_id
         self.streams[stream_id].end_stream()
@@ -188,9 +149,6 @@ class H2ClientProtocol(Protocol):
     def stream_reset(self, event: StreamReset):
         pass
 
-    def trailers_received(self, event: TrailersReceived):
-        pass
-
     def window_updated(self, event: WindowUpdated):
         stream_id = event.stream_id
         if stream_id != 0:
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index ab5a0fc88..b37755042 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -1,5 +1,6 @@
 from urllib.parse import urlparse
 
+from h2.connection import H2Connection
 from twisted.internet.defer import Deferred
 
 from scrapy.http import Request, Response
@@ -17,7 +18,7 @@ class Stream:
     1. Combine all the data frames
     """
 
-    def __init__(self, stream_id: int, request: Request, connection):
+    def __init__(self, stream_id: int, request: Request, connection: H2Connection):
         """
         Arguments:
             stream_id {int} -- For one HTTP/2 connection each stream is
@@ -27,7 +28,7 @@ class Stream:
         """
         self.stream_id = stream_id
         self._request = request
-        self._client_protocol = connection
+        self._conn = connection
 
         self._request_body = self._request.body
         self.content_length = 0 if self._request_body is None else len(self._request_body)
@@ -70,13 +71,12 @@ class Stream:
             # TODO: Check if scheme can be "http" for HTTP/2 ?
             (":scheme", "https"),
             (":path", url.path),
-            # ("Content-Length", str(self.content_length))
 
             # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers
             #  are sent for request having body
         ]
 
-        self._client_protocol.send_headers(self.stream_id, http2_request_headers)
+        self._conn.send_headers(self.stream_id, http2_request_headers, end_stream=False)
         self.send_data()
 
     def send_data(self):
@@ -95,10 +95,10 @@ class Stream:
         #    3.2 Small number of requests
 
         # Firstly, check what the flow control window is for current stream.
-        window_size = self._client_protocol.conn.local_flow_control_window(stream_id=self.stream_id)
+        window_size = self._conn.local_flow_control_window(stream_id=self.stream_id)
 
         # Next, check what the maximum frame size is.
-        max_frame_size = self._client_protocol.conn.max_outbound_frame_size
+        max_frame_size = self._conn.max_outbound_frame_size
 
         # We will send no more than the window size or the remaining file size
         # of data in this call, whichever is smaller.
@@ -111,14 +111,14 @@ class Stream:
             data_chunk_start = self.content_length - self.remaining_content_length
             data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size]
 
-            self._client_protocol.send_data(self.stream_id, data_chunk, end_stream=False)
+            self._conn.send_data(self.stream_id, data_chunk, end_stream=False)
 
             bytes_to_send = max(0, bytes_to_send - chunk_size)
             self.remaining_content_length = max(0, self.remaining_content_length - chunk_size)
 
         # End the stream if no more data has to be send
         if self.remaining_content_length == 0:
-            self._client_protocol.end_stream(self.stream_id)
+            self._conn.end_stream(self.stream_id)
         else:
             # TODO: Continue from here :)
             pass

From de4a34365a2d872eb69ab2a2edfb723658e28530 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sun, 14 Jun 2020 22:40:49 +0530
Subject: [PATCH 047/716] fix: large data chunk not received

Every data chunk received needs to be acknowledged to
- update the flow control window size
- get furthur data chunks from the server
---
 scrapy/core/http2/protocol.py | 74 +++++++++++++++++++++---------
 scrapy/core/http2/stream.py   | 85 ++++++++++++++++++++++++++++++-----
 2 files changed, 129 insertions(+), 30 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 4036dfb3e..167596340 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -1,10 +1,10 @@
 import logging
-from typing import Dict, List
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
 from h2.events import (
-    ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, StreamReset, WindowUpdated
+    ConnectionTerminated, DataReceived, ResponseReceived,
+    StreamEnded, StreamReset, WindowUpdated
 )
 from twisted.internet.protocol import connectionDone, Protocol
 
@@ -18,33 +18,57 @@ class H2ClientProtocol(Protocol):
     # TODO:
     #  1. Check for user-agent while testing
     #  2. Add support for cookies
-    #  3. Handle priority updates
+    #  3. Handle priority updates (Not required)
+    #  4. Handle case when received events have StreamID = 0 (applied to H2Connection)
+    #  1 & 2:
+    #   - Automatically handled by the Request middleware
+    #   - request.headers will have 'Set-Cookie' value
 
     def __init__(self):
         config = H2Configuration(client_side=True, header_encoding='utf-8')
         self.conn = H2Connection(config=config)
 
+        # Address of the server we are connected to
+        # these are updated when connection is successfully made
+        self.destination = None
+
         # ID of the next request stream
-        # Assuming each request stream creates a new response stream
-        # we increment by 2 for each new request stream created
+        # Following the convention made by hyper-h2 each client ID
+        # will be odd.
         self.next_stream_id = 1
 
         # Streams are stored in a dictionary keyed off their stream IDs
-        self.streams: Dict[int, Stream] = {}
+        self.streams = {}
 
         # Boolean to keep track the connection is made
         # If requests are received before connection is made
         # we keep all requests in a pool and send them as the connection
         # is made
         self.is_connection_made = False
-        self._pending_request_stream_pool: List[Stream] = []
+        self._pending_request_stream_pool = []
+
+    def _stream_close_cb(self, stream_id: int):
+        """Called when stream is closed completely
+        """
+        try:
+            del self.streams[stream_id]
+        except KeyError:
+            pass
 
     def _new_stream(self, request: Request):
         """Instantiates a new Stream object
         """
-        stream = Stream(self.next_stream_id, request, self.conn)
+        stream_id = self.next_stream_id
         self.next_stream_id += 2
 
+        stream = Stream(
+            stream_id=stream_id,
+            request=request,
+            connection=self.conn,
+            write_to_transport=self._write_to_transport,
+            cb_close=lambda: self._stream_close_cb(stream_id)
+        )
+
         self.streams[stream.stream_id] = stream
         return stream
 
@@ -53,7 +77,6 @@ class H2ClientProtocol(Protocol):
         # Initiate all pending requests
         for stream in self._pending_request_stream_pool:
             stream.initiate_request()
-            self._write_to_transport()
 
         self._pending_request_stream_pool.clear()
 
@@ -81,13 +104,15 @@ class H2ClientProtocol(Protocol):
         """Called by Twisted when the connection is established. We can start
         sending some data now: we should open with the connection preamble.
         """
+        self.destination = self.transport.connector.getDestination()
+        LOGGER.info('Connection made to {}'.format(self.destination))
+
         self.conn.initiate_connection()
         self._write_to_transport()
+        self.is_connection_made = True
 
         self._send_pending_requests()
 
-        self.is_connection_made = True
-
     def dataReceived(self, data):
         events = self.conn.receive_data(data)
         self._handle_events(events)
@@ -95,15 +120,18 @@ class H2ClientProtocol(Protocol):
 
     def connectionLost(self, reason=connectionDone):
         """Called by Twisted when the transport connection is lost.
+        No need to write anything to transport here.
         """
-        stream_ids = list(self.streams.keys())
+        # Pop all streams which were pending and were not yet started
+        for stream_id in list(self.streams):
+            try:
+                self.streams[stream_id].lost_connection()
+            except KeyError:
+                pass
 
-        for stream in self._pending_request_stream_pool:
-            stream_ids.remove(stream.stream_id)
+        self.conn.close_connection()
 
-        for stream_id in stream_ids:
-            # TODO: Close each Stream instance in a clean manner
-            self.conn.end_stream(stream_id)
+        LOGGER.info("Connection lost with reason " + str(reason))
 
     def _handle_events(self, events):
         """Private method which acts as a bridge between the events
@@ -136,7 +164,7 @@ class H2ClientProtocol(Protocol):
 
     def data_received(self, event: DataReceived):
         stream_id = event.stream_id
-        self.streams[stream_id].receive_data(event.data)
+        self.streams[stream_id].receive_data(event.data, event.flow_controlled_length)
 
     def response_received(self, event: ResponseReceived):
         stream_id = event.stream_id
@@ -147,9 +175,15 @@ class H2ClientProtocol(Protocol):
         self.streams[stream_id].end_stream()
 
     def stream_reset(self, event: StreamReset):
-        pass
+        # TODO: event.stream_id was abruptly closed
+        #  Q. What should be the response? (Failure/Partial/???)
+        self.streams[event.stream_id].reset()
 
     def window_updated(self, event: WindowUpdated):
         stream_id = event.stream_id
         if stream_id != 0:
-            self.streams[stream_id].window_updated()
+            self.streams[stream_id].receive_window_update(event.delta)
+        else:
+            # TODO:
+            #  Q. What to do when StreamID=0 ?
+            pass
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index b37755042..7c9cf7cf5 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -1,3 +1,4 @@
+import logging
 from urllib.parse import urlparse
 
 from h2.connection import H2Connection
@@ -6,6 +7,8 @@ from twisted.internet.defer import Deferred
 from scrapy.http import Request, Response
 from scrapy.http.headers import Headers
 
+LOGGER = logging.getLogger(__name__)
+
 
 class Stream:
     """Represents a single HTTP/2 Stream.
@@ -18,17 +21,30 @@ class Stream:
     1. Combine all the data frames
     """
 
-    def __init__(self, stream_id: int, request: Request, connection: H2Connection):
+    def __init__(
+            self,
+            stream_id: int,
+            request: Request,
+            connection: H2Connection,
+            write_to_transport,
+            cb_close
+    ):
         """
         Arguments:
             stream_id {int} -- For one HTTP/2 connection each stream is
                 uniquely identified by a single integer
             request {Request} -- HTTP request
             connection {H2Connection} -- HTTP/2 connection this stream belongs to.
+            write_to_transport {callable} -- Method used to write & send data to the server
+                This method should be used whenever some frame is to be sent to the server.
+            cb_close {callable} -- Method called when this stream is closed
+                to notify the TCP connection instance.
         """
         self.stream_id = stream_id
         self._request = request
         self._conn = connection
+        self._write_to_transport = write_to_transport
+        self._cb_close = cb_close
 
         self._request_body = self._request.body
         self.content_length = 0 if self._request_body is None else len(self._request_body)
@@ -36,13 +52,20 @@ class Stream:
         # Each time we send a data frame, we will decrease value by the amount send.
         self.remaining_content_length = self.content_length
 
-        # Flag to keep track whether we have ended this stream
-        self.stream_ended = True
+        # Flag to keep track whether we have closed this stream
+        self.stream_closed_local = False
+
+        # Flag to keep track whether the server has closed the stream
+        self.stream_closed_server = False
 
         # Data received frame by frame from the server is appended
         # and passed to the response Deferred when completely received.
         self._response_data = b""
 
+        # The amount of data received that counts against the flow control
+        # window
+        self._response_flow_controlled_size = 0
+
         # Headers received after sending the request
         self._response_headers = Headers({})
 
@@ -77,6 +100,8 @@ class Stream:
         ]
 
         self._conn.send_headers(self.stream_id, http2_request_headers, end_stream=False)
+        self._write_to_transport()
+
         self.send_data()
 
     def send_data(self):
@@ -112,32 +137,59 @@ class Stream:
             data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size]
 
             self._conn.send_data(self.stream_id, data_chunk, end_stream=False)
+            self._write_to_transport()
 
             bytes_to_send = max(0, bytes_to_send - chunk_size)
             self.remaining_content_length = max(0, self.remaining_content_length - chunk_size)
 
         # End the stream if no more data has to be send
         if self.remaining_content_length == 0:
+            self.stream_closed_local = True
             self._conn.end_stream(self.stream_id)
-        else:
-            # TODO: Continue from here :)
-            pass
 
-    def window_updated(self):
+        self._write_to_transport()
+
+        # Q. What about the rest of the data?
+        # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame
+
+    def receive_window_update(self, delta):
         """Flow control window size was changed.
         Send data that earlier could not be sent as we were
         blocked behind the flow control.
+
+        Arguments:
+            delta -- Window change delta
         """
-        if self.remaining_content_length > 0 and not self.stream_ended:
+        if self.remaining_content_length > 0 and not self.stream_closed_local:
             self.send_data()
 
-    def receive_data(self, data: bytes):
+    def receive_data(self, data: bytes, flow_controlled_length: int):
         self._response_data += data
+        self._response_flow_controlled_size += flow_controlled_length
+
+        # Acknowledge the data received
+        self._conn.acknowledge_received_data(
+            self._response_flow_controlled_size,
+            self.stream_id
+        )
 
     def receive_headers(self, headers):
         for name, value in headers:
             self._response_headers[name] = value
 
+    def reset(self):
+        """Received a RST_STREAM -- forcefully reset"""
+        # TODO:
+        #  Q1. Do we need to send the request again?
+        #  Q2. What response should we send now?
+        self.stream_closed_server = True
+        self._cb_close()
+
+    def lost_connection(self):
+        # TODO: Same as self.reset
+        self.stream_closed_server = True
+        self._cb_close()
+
     def end_stream(self):
         """Stream is ended by the server hence no further
         data or headers should be expected on this stream.
@@ -145,7 +197,20 @@ class Stream:
         We will call the response deferred callback passing
         the response object
         """
-        # TODO: Set flags, certificate, ip_address
+        assert self.stream_closed_server is False
+        self.stream_closed_server = True
+
+        self._fire_response_deferred()
+        self._cb_close()
+
+    def _fire_response_deferred(self):
+        # TODO:
+        #  1. Set flags, certificate, ip_address in response
+        #  2. Should we fire this in case of
+        #   2.1 StreamReset in between when data is received partially
+        #   2.2 Forcefully closed the stream
+
+        # NOTE: Presently on fired with successful response
         response = Response(
             url=self._request.url,
             status=self._response_headers[":status"],

From 01ad8b31ab7fe86fd78a70b09a6dd61a497e0ccb Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Mon, 15 Jun 2020 05:14:00 +0530
Subject: [PATCH 048/716] refactor(http2): clean up

- make separate function to parse http headers from Request instance
---
 scrapy/core/http2/protocol.py | 12 ++------
 scrapy/core/http2/stream.py   | 55 ++++++++++++++++++-----------------
 2 files changed, 32 insertions(+), 35 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 167596340..915284c33 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -50,10 +50,7 @@ class H2ClientProtocol(Protocol):
     def _stream_close_cb(self, stream_id: int):
         """Called when stream is closed completely
         """
-        try:
-            del self.streams[stream_id]
-        except KeyError:
-            pass
+        self.streams.pop(stream_id, None)
 
     def _new_stream(self, request: Request):
         """Instantiates a new Stream object
@@ -66,7 +63,7 @@ class H2ClientProtocol(Protocol):
             request=request,
             connection=self.conn,
             write_to_transport=self._write_to_transport,
-            cb_close=lambda: self._stream_close_cb(stream_id)
+            cb_close=self._stream_close_cb
         )
 
         self.streams[stream.stream_id] = stream
@@ -124,10 +121,7 @@ class H2ClientProtocol(Protocol):
         """
         # Pop all streams which were pending and were not yet started
         for stream_id in list(self.streams):
-            try:
-                self.streams[stream_id].lost_connection()
-            except KeyError:
-                pass
+            self.streams[stream_id].lost_connection()
 
         self.conn.close_connection()
 
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 7c9cf7cf5..a0b75850d 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -81,25 +81,27 @@ class Stream:
         """
         return self._deferred_response
 
-    def initiate_request(self):
-        http2_request_headers = []
-        for name, value in self._request.headers.items():
-            http2_request_headers.append((name, value))
-
+    def _get_request_headers(self):
         url = urlparse(self._request.url)
-        http2_request_headers += [
-            (":method", self._request.method),
-            (":authority", url.netloc),
 
-            # TODO: Check if scheme can be "http" for HTTP/2 ?
-            (":scheme", "https"),
-            (":path", url.path),
+        # Make sure pseudo-headers comes before all the other headers
+        headers = [
+            (':method', self._request.method),
+            (':authority', url.netloc),
 
-            # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers
-            #  are sent for request having body
+            # TODO: Check if scheme can be 'http' for HTTP/2 ?
+            (':scheme', 'https'),
+            (':path', url.path),
         ]
 
-        self._conn.send_headers(self.stream_id, http2_request_headers, end_stream=False)
+        for name, value in self._request.headers.items():
+            headers.append((name, value[0]))
+
+        return headers
+
+    def initiate_request(self):
+        headers = self._get_request_headers()
+        self._conn.send_headers(self.stream_id, headers, end_stream=False)
         self._write_to_transport()
 
         self.send_data()
@@ -127,23 +129,24 @@ class Stream:
 
         # We will send no more than the window size or the remaining file size
         # of data in this call, whichever is smaller.
-        bytes_to_send = min(window_size, self.remaining_content_length)
+        bytes_to_send_size = min(window_size, self.remaining_content_length)
 
         # We now need to send a number of data frames.
-        while bytes_to_send > 0:
-            chunk_size = min(bytes_to_send, max_frame_size)
+        while bytes_to_send_size > 0:
+            chunk_size = min(bytes_to_send_size, max_frame_size)
 
-            data_chunk_start = self.content_length - self.remaining_content_length
-            data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size]
+            data_chunk_start_id = self.content_length - self.remaining_content_length
+            data_chunk = self._request_body[data_chunk_start_id:data_chunk_start_id + chunk_size]
 
             self._conn.send_data(self.stream_id, data_chunk, end_stream=False)
-            self._write_to_transport()
 
-            bytes_to_send = max(0, bytes_to_send - chunk_size)
-            self.remaining_content_length = max(0, self.remaining_content_length - chunk_size)
+            bytes_to_send_size = bytes_to_send_size - chunk_size
+            self.remaining_content_length = self.remaining_content_length - chunk_size
 
         # End the stream if no more data has to be send
-        if self.remaining_content_length == 0:
+        if self.remaining_content_length <= 0:
+            self.remaining_content_length = 0
+
             self.stream_closed_local = True
             self._conn.end_stream(self.stream_id)
 
@@ -183,12 +186,12 @@ class Stream:
         #  Q1. Do we need to send the request again?
         #  Q2. What response should we send now?
         self.stream_closed_server = True
-        self._cb_close()
+        self._cb_close(self.stream_id)
 
     def lost_connection(self):
         # TODO: Same as self.reset
         self.stream_closed_server = True
-        self._cb_close()
+        self._cb_close(self.stream_id)
 
     def end_stream(self):
         """Stream is ended by the server hence no further
@@ -201,7 +204,7 @@ class Stream:
         self.stream_closed_server = True
 
         self._fire_response_deferred()
-        self._cb_close()
+        self._cb_close(self.stream_id)
 
     def _fire_response_deferred(self):
         # TODO:

From 089dbc75e78a2da9c455f21bb3c7ebaaeb2e3582 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 17 Jun 2020 20:57:03 +0530
Subject: [PATCH 049/716] chore: use deque for pending request pool

- Use itertools.count to generate next stream_id

BREAKING CHANGES
When sending data/body more than the local flow control window -- no
window update occurs to send the remaining data frames. Hence, the
complete body is not send resulting in no response received.
---
 scrapy/core/http2/protocol.py | 16 +++++++++-------
 scrapy/core/http2/stream.py   |  4 ++--
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 915284c33..dbc048ffa 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -1,4 +1,6 @@
+import itertools
 import logging
+from collections import deque
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
@@ -35,7 +37,7 @@ class H2ClientProtocol(Protocol):
         # ID of the next request stream
         # Following the convention made by hyper-h2 each client ID
         # will be odd.
-        self.next_stream_id = 1
+        self.stream_id_count = itertools.count(start=1, step=2)
 
         # Streams are stored in a dictionary keyed off their stream IDs
         self.streams = {}
@@ -45,7 +47,7 @@ class H2ClientProtocol(Protocol):
         # we keep all requests in a pool and send them as the connection
         # is made
         self.is_connection_made = False
-        self._pending_request_stream_pool = []
+        self._pending_request_stream_pool = deque()
 
     def _stream_close_cb(self, stream_id: int):
         """Called when stream is closed completely
@@ -55,8 +57,7 @@ class H2ClientProtocol(Protocol):
     def _new_stream(self, request: Request):
         """Instantiates a new Stream object
         """
-        stream_id = self.next_stream_id
-        self.next_stream_id += 2
+        stream_id = next(self.stream_id_count)
 
         stream = Stream(
             stream_id=stream_id,
@@ -72,11 +73,10 @@ class H2ClientProtocol(Protocol):
     def _send_pending_requests(self):
         # TODO: handle MAX_CONCURRENT_STREAMS
         # Initiate all pending requests
-        for stream in self._pending_request_stream_pool:
+        while len(self._pending_request_stream_pool):
+            stream = self._pending_request_stream_pool.popleft()
             stream.initiate_request()
 
-        self._pending_request_stream_pool.clear()
-
     def _write_to_transport(self):
         """ Write data to the underlying transport connection
         from the HTTP2 connection instance if any
@@ -108,6 +108,8 @@ class H2ClientProtocol(Protocol):
         self._write_to_transport()
         self.is_connection_made = True
 
+        # Send off all the pending requests
+        # as now we have established a proper HTTP/2 connection
         self._send_pending_requests()
 
     def dataReceived(self, data):
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index a0b75850d..c2e1adce5 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -132,7 +132,7 @@ class Stream:
         bytes_to_send_size = min(window_size, self.remaining_content_length)
 
         # We now need to send a number of data frames.
-        while bytes_to_send_size > 0:
+        while bytes_to_send_size:
             chunk_size = min(bytes_to_send_size, max_frame_size)
 
             data_chunk_start_id = self.content_length - self.remaining_content_length
@@ -163,7 +163,7 @@ class Stream:
         Arguments:
             delta -- Window change delta
         """
-        if self.remaining_content_length > 0 and not self.stream_closed_local:
+        if self.stream_closed_local is False:
             self.send_data()
 
     def receive_data(self, data: bytes, flow_controlled_length: int):

From 700df3eeb7c540892c8b2910db688f1cc5d1c845 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 17 Jun 2020 21:02:14 +0530
Subject: [PATCH 050/716] test: mockserver with h2 protocol for tests

- add Twisted[http2] in setup.py requirements
- add test_protocol.py to test the current implementation

BREAKING CHANGES
test_download times out because of no protocol negotiated between
Mockserver and HTTP/2 client
---
 scrapy/core/http2/test_protocol.py  | 67 +++++++++++++++++++++++++++
 setup.py                            |  6 +--
 tests/test_http2_client_protocol.py | 71 +++++++++++++++++++++++++++++
 3 files changed, 141 insertions(+), 3 deletions(-)
 create mode 100644 scrapy/core/http2/test_protocol.py
 create mode 100644 tests/test_http2_client_protocol.py

diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.py
new file mode 100644
index 000000000..c7782a518
--- /dev/null
+++ b/scrapy/core/http2/test_protocol.py
@@ -0,0 +1,67 @@
+# This is simple script to test
+
+import json
+
+from twisted.internet import reactor
+from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
+from twisted.internet.ssl import optionsForClientTLS
+
+from scrapy.core.http2.protocol import H2ClientProtocol
+from scrapy.http import Request, Response, JsonRequest
+
+try:
+    with open('data.json', 'r') as f:
+        JSON_DATA = json.load(f)
+except:
+    JSON_DATA = {
+        "data": "To test for really large amount of data -- Add data.json with lots of data.",
+        "why": "To test whether correct data is sent :)"
+    }
+
+# Use nghttp2 for testing whether basic setup works - for small response
+HTTPBIN_AUTHORITY = u'nghttp2.org'
+HTTPBIN_REQUEST_URLS = 1 * [
+    Request(url='https://nghttp2.org/httpbin/get', method='GET'),
+    Request(url='https://nghttp2.org/httpbin/post', method='POST'),
+    JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA),
+]
+
+# Use POKE_API for testing large responses
+POKE_API_AUTHORITY = u'pokeapi.co'
+POKE_API_REQUESTS = 15 * [
+    Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'),
+    Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'),
+    Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'),
+    Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'),  # should give 404
+]
+
+AUTHORITY = POKE_API_AUTHORITY
+REQUEST_URLS = POKE_API_REQUESTS
+
+options = optionsForClientTLS(
+    hostname=AUTHORITY,
+    acceptableProtocols=[b'h2'],
+)
+
+protocol = H2ClientProtocol()
+
+count_responses = 1
+
+
+def print_response(response):
+    global count_responses
+    assert isinstance(response, Response)
+    print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body)))
+    count_responses = count_responses + 1
+
+
+for request in REQUEST_URLS:
+    d = protocol.request(request)
+    d.addCallback(print_response)
+
+connectProtocol(
+    SSL4ClientEndpoint(reactor, AUTHORITY, 443, options),
+    protocol
+)
+
+reactor.run()
diff --git a/setup.py b/setup.py
index 1b3c6771a..dafa5684a 100644
--- a/setup.py
+++ b/setup.py
@@ -1,8 +1,8 @@
 from os.path import dirname, join
+
 from pkg_resources import parse_version
 from setuptools import setup, find_packages, __version__ as setuptools_version
 
-
 with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f:
     version = f.read().decode('ascii').strip()
 
@@ -25,12 +25,11 @@ if has_environment_marker_platform_impl_support():
         'PyPyDispatcher>=2.1.0',
     ]
 
-
 setup(
     name='Scrapy',
     version=version,
     url='https://scrapy.org',
-    project_urls = {
+    project_urls={
         'Documentation': 'https://docs.scrapy.org/',
         'Source': 'https://github.com/scrapy/scrapy',
         'Tracker': 'https://github.com/scrapy/scrapy/issues',
@@ -69,6 +68,7 @@ setup(
     python_requires='>=3.5',
     install_requires=[
         'Twisted>=17.9.0',
+        'Twisted[http2]>=17.9.0'
         'cryptography>=2.0',
         'cssselect>=0.9.1',
         'lxml>=3.5.0',
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
new file mode 100644
index 000000000..7830f7028
--- /dev/null
+++ b/tests/test_http2_client_protocol.py
@@ -0,0 +1,71 @@
+import os
+import shutil
+
+from twisted.internet import defer, reactor
+from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
+from twisted.internet.ssl import optionsForClientTLS
+from twisted.protocols.policies import WrappingFactory
+from twisted.python.filepath import FilePath
+from twisted.trial import unittest
+from twisted.web import static, server
+
+from scrapy.core.http2.protocol import H2ClientProtocol
+from scrapy.http import Request
+from tests.mockserver import ssl_context_factory
+
+
+class Http2ClientProtocolTestCase(unittest.TestCase):
+    scheme = 'https'
+
+    # only used for HTTPS tests
+    file_key = 'keys/localhost.key'
+    file_certificate = 'keys/localhost.crt'
+
+    def setUp(self):
+        # Start server for testing
+        self.path_temp = self.mktemp()
+        os.mkdir(self.path_temp)
+        FilePath(self.path_temp).child('file').setContent(b"0123456789")
+        r = static.File(self.path_temp)
+
+        self.site = server.Site(r, timeout=None)
+        self.wrapper = WrappingFactory(self.site)
+        self.host = 'localhost'
+        if self.scheme is 'https':
+            self.port = reactor.listenSSL(
+                0, self.wrapper,
+                ssl_context_factory(self.file_key, self.file_certificate),
+                interface=self.host
+            )
+        else:
+            self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
+
+        self.port_number = self.port.getHost().port
+
+        # Connect to the server using the custom HTTP2ClientProtocol
+        options = optionsForClientTLS(
+            hostname=self.host,
+            acceptableProtocols=[b'h2']
+        )
+
+        self.protocol = H2ClientProtocol()
+
+        connectProtocol(
+            endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options),
+            protocol=self.protocol
+        )
+
+    def getURL(self, path):
+        return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path)
+
+    @defer.inlineCallbacks
+    def tearDown(self):
+        yield self.port.stopListening()
+        shutil.rmtree(self.path_temp)
+
+    def test_download(self):
+        request = Request(self.getURL('file'))
+        d = self.protocol.request(request)
+        d.addCallback(lambda response: response.body)
+        d.addCallback(self.assertEqual, b"0123456789")
+        return d

From 303485a9b4fd86c5123c96c81a9401b5e323a91d Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sun, 21 Jun 2020 00:33:34 +0530
Subject: [PATCH 051/716] fix(http2): POST request not sending large body

---
 scrapy/core/http2/protocol.py |  51 +++++++-----
 scrapy/core/http2/stream.py   | 148 ++++++++++++++++++++++++----------
 2 files changed, 138 insertions(+), 61 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index dbc048ffa..d6134183a 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -1,3 +1,4 @@
+import ipaddress
 import itertools
 import logging
 from collections import deque
@@ -5,7 +6,7 @@ from collections import deque
 from h2.config import H2Configuration
 from h2.connection import H2Connection
 from h2.events import (
-    ConnectionTerminated, DataReceived, ResponseReceived,
+    DataReceived, ResponseReceived, SettingsAcknowledged,
     StreamEnded, StreamReset, WindowUpdated
 )
 from twisted.internet.protocol import connectionDone, Protocol
@@ -49,6 +50,13 @@ class H2ClientProtocol(Protocol):
         self.is_connection_made = False
         self._pending_request_stream_pool = deque()
 
+        # Some meta data of this connection
+        # initialized when connection is successfully made
+        self._metadata = {
+            'certificate': None,
+            'ip_address': None
+        }
+
     def _stream_close_cb(self, stream_id: int):
         """Called when stream is closed completely
         """
@@ -63,6 +71,7 @@ class H2ClientProtocol(Protocol):
             stream_id=stream_id,
             request=request,
             connection=self.conn,
+            metadata=self._metadata,
             write_to_transport=self._write_to_transport,
             cb_close=self._stream_close_cb
         )
@@ -73,7 +82,7 @@ class H2ClientProtocol(Protocol):
     def _send_pending_requests(self):
         # TODO: handle MAX_CONCURRENT_STREAMS
         # Initiate all pending requests
-        while len(self._pending_request_stream_pool):
+        while self._pending_request_stream_pool:
             stream = self._pending_request_stream_pool.popleft()
             stream.initiate_request()
 
@@ -84,6 +93,8 @@ class H2ClientProtocol(Protocol):
         data = self.conn.data_to_send()
         self.transport.write(data)
 
+        LOGGER.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address']))
+
     def request(self, _request: Request):
         stream = self._new_stream(_request)
         d = stream.get_response()
@@ -101,17 +112,16 @@ class H2ClientProtocol(Protocol):
         """Called by Twisted when the connection is established. We can start
         sending some data now: we should open with the connection preamble.
         """
-        self.destination = self.transport.connector.getDestination()
+        self.destination = self.transport.getPeer()
         LOGGER.info('Connection made to {}'.format(self.destination))
 
+        self._metadata['certificate'] = self.transport.getPeerCertificate()
+        self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host)
+
         self.conn.initiate_connection()
         self._write_to_transport()
         self.is_connection_made = True
 
-        # Send off all the pending requests
-        # as now we have established a proper HTTP/2 connection
-        self._send_pending_requests()
-
     def dataReceived(self, data):
         events = self.conn.receive_data(data)
         self._handle_events(events)
@@ -123,7 +133,7 @@ class H2ClientProtocol(Protocol):
         """
         # Pop all streams which were pending and were not yet started
         for stream_id in list(self.streams):
-            self.streams[stream_id].lost_connection()
+            self.streams[stream_id].close()
 
         self.conn.close_connection()
 
@@ -139,9 +149,7 @@ class H2ClientProtocol(Protocol):
         """
         for event in events:
             LOGGER.debug(event)
-            if isinstance(event, ConnectionTerminated):
-                self.connection_terminated(event)
-            elif isinstance(event, DataReceived):
+            if isinstance(event, DataReceived):
                 self.data_received(event)
             elif isinstance(event, ResponseReceived):
                 self.response_received(event)
@@ -151,13 +159,12 @@ class H2ClientProtocol(Protocol):
                 self.stream_reset(event)
             elif isinstance(event, WindowUpdated):
                 self.window_updated(event)
+            elif isinstance(event, SettingsAcknowledged):
+                self.settings_acknowledged(event)
             else:
                 LOGGER.info("Received unhandled event {}".format(event))
 
     # Event handler functions starts here
-    def connection_terminated(self, event: ConnectionTerminated):
-        pass
-
     def data_received(self, event: DataReceived):
         stream_id = event.stream_id
         self.streams[stream_id].receive_data(event.data, event.flow_controlled_length)
@@ -166,20 +173,26 @@ class H2ClientProtocol(Protocol):
         stream_id = event.stream_id
         self.streams[stream_id].receive_headers(event.headers)
 
+    def settings_acknowledged(self, event: SettingsAcknowledged):
+        # Send off all the pending requests
+        # as now we have established a proper HTTP/2 connection
+        self._send_pending_requests()
+
     def stream_ended(self, event: StreamEnded):
         stream_id = event.stream_id
-        self.streams[stream_id].end_stream()
+        self.streams[stream_id].close()
 
     def stream_reset(self, event: StreamReset):
         # TODO: event.stream_id was abruptly closed
         #  Q. What should be the response? (Failure/Partial/???)
-        self.streams[event.stream_id].reset()
+        self.streams[event.stream_id].close(event)
 
     def window_updated(self, event: WindowUpdated):
         stream_id = event.stream_id
         if stream_id != 0:
             self.streams[stream_id].receive_window_update(event.delta)
         else:
-            # TODO:
-            #  Q. What to do when StreamID=0 ?
-            pass
+            # Send leftover data for all the streams
+            for stream in self.streams.values():
+                if stream.request_sent:
+                    stream.send_data()
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index c2e1adce5..e8b4471d6 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -1,11 +1,15 @@
 import logging
+from typing import Dict
 from urllib.parse import urlparse
 
 from h2.connection import H2Connection
+from h2.events import StreamEnded
+from h2.exceptions import StreamClosedError
 from twisted.internet.defer import Deferred
 
-from scrapy.http import Request, Response
+from scrapy.http import Request
 from scrapy.http.headers import Headers
+from scrapy.responsetypes import responsetypes
 
 LOGGER = logging.getLogger(__name__)
 
@@ -22,12 +26,13 @@ class Stream:
     """
 
     def __init__(
-            self,
-            stream_id: int,
-            request: Request,
-            connection: H2Connection,
-            write_to_transport,
-            cb_close
+        self,
+        stream_id: int,
+        request: Request,
+        connection: H2Connection,
+        metadata: Dict,
+        write_to_transport,
+        cb_close
     ):
         """
         Arguments:
@@ -35,6 +40,7 @@ class Stream:
                 uniquely identified by a single integer
             request {Request} -- HTTP request
             connection {H2Connection} -- HTTP/2 connection this stream belongs to.
+            metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection
             write_to_transport {callable} -- Method used to write & send data to the server
                 This method should be used whenever some frame is to be sent to the server.
             cb_close {callable} -- Method called when this stream is closed
@@ -43,12 +49,16 @@ class Stream:
         self.stream_id = stream_id
         self._request = request
         self._conn = connection
+        self._metadata = metadata
         self._write_to_transport = write_to_transport
         self._cb_close = cb_close
 
         self._request_body = self._request.body
         self.content_length = 0 if self._request_body is None else len(self._request_body)
 
+        # Flag to keep track whether this stream has initiated the request
+        self.request_sent = False
+
         # Each time we send a data frame, we will decrease value by the amount send.
         self.remaining_content_length = self.content_length
 
@@ -58,20 +68,30 @@ class Stream:
         # Flag to keep track whether the server has closed the stream
         self.stream_closed_server = False
 
-        # Data received frame by frame from the server is appended
-        # and passed to the response Deferred when completely received.
-        self._response_data = b""
-
         # The amount of data received that counts against the flow control
         # window
         self._response_flow_controlled_size = 0
 
-        # Headers received after sending the request
-        self._response_headers = Headers({})
+        # Private variable used to build the response
+        # this response is then converted to appropriate Response class
+        # passed to the response deferred callback
+        self._response = {
+            # Data received frame by frame from the server is appended
+            # and passed to the response Deferred when completely received.
+            'body': b'',
+
+            # Headers received after sending the request
+            'headers': Headers({})
+        }
 
         # TODO: Add canceller for the Deferred below
         self._deferred_response = Deferred()
 
+    def __str__(self):
+        return "Stream(id={})".format(self.stream_id)
+
+    __repr__ = __str__
+
     def get_response(self):
         """Simply return a Deferred which fires when response
         from the asynchronous request is available
@@ -104,6 +124,8 @@ class Stream:
         self._conn.send_headers(self.stream_id, headers, end_stream=False)
         self._write_to_transport()
 
+        self.request_sent = True
+
         self.send_data()
 
     def send_data(self):
@@ -112,7 +134,18 @@ class Stream:
 
          If the content length is 0 initially then we end the stream immediately and
          wait for response data.
+
+         Warning: Only call this method when stream not closed from client side
+            and has initiated request already by sending HEADER frame. If not then
+            stream will be closed from client side with 499 response.
+
+            TODO: Q. Should we instead raise ProtocolError here with a proper message?
          """
+        if self.stream_closed_local or self.stream_closed_server:
+            raise StreamClosedError(self.stream_id)
+        elif not self.request_sent:
+            self.close()
+            return
 
         # TODO:
         #  1. Add test for sending very large data
@@ -132,7 +165,8 @@ class Stream:
         bytes_to_send_size = min(window_size, self.remaining_content_length)
 
         # We now need to send a number of data frames.
-        while bytes_to_send_size:
+        data_frames_sent = 0
+        while bytes_to_send_size > 0:
             chunk_size = min(bytes_to_send_size, max_frame_size)
 
             data_chunk_start_id = self.content_length - self.remaining_content_length
@@ -140,16 +174,24 @@ class Stream:
 
             self._conn.send_data(self.stream_id, data_chunk, end_stream=False)
 
+            data_frames_sent += 1
             bytes_to_send_size = bytes_to_send_size - chunk_size
             self.remaining_content_length = self.remaining_content_length - chunk_size
 
-        # End the stream if no more data has to be send
-        if self.remaining_content_length <= 0:
-            self.remaining_content_length = 0
+        self.remaining_content_length = max(0, self.remaining_content_length)
+        LOGGER.debug("{} sending {}/{} data bytes ({} frames) to {}".format(
+            self,
+            self.content_length - self.remaining_content_length, self.content_length,
+            data_frames_sent,
+            self._metadata['ip_address'])
+        )
 
+        # End the stream if no more data needs to be send
+        if self.remaining_content_length == 0:
             self.stream_closed_local = True
             self._conn.end_stream(self.stream_id)
 
+        # Write data to transport -- Empty the outstanding data
         self._write_to_transport()
 
         # Q. What about the rest of the data?
@@ -163,11 +205,11 @@ class Stream:
         Arguments:
             delta -- Window change delta
         """
-        if self.stream_closed_local is False:
+        if self.remaining_content_length > 0 and not self.stream_closed_server:
             self.send_data()
 
     def receive_data(self, data: bytes, flow_controlled_length: int):
-        self._response_data += data
+        self._response['body'] += data
         self._response_flow_controlled_size += flow_controlled_length
 
         # Acknowledge the data received
@@ -178,47 +220,69 @@ class Stream:
 
     def receive_headers(self, headers):
         for name, value in headers:
-            self._response_headers[name] = value
+            self._response['headers'][name] = value
 
-    def reset(self):
-        """Received a RST_STREAM -- forcefully reset"""
-        # TODO:
-        #  Q1. Do we need to send the request again?
-        #  Q2. What response should we send now?
-        self.stream_closed_server = True
-        self._cb_close(self.stream_id)
+    def close(self, event=None):
+        """Based on the event sent we will handle each case.
 
-    def lost_connection(self):
-        # TODO: Same as self.reset
-        self.stream_closed_server = True
-        self._cb_close(self.stream_id)
-
-    def end_stream(self):
-        """Stream is ended by the server hence no further
+        event: StreamEnded
+        Stream is ended by the server hence no further
         data or headers should be expected on this stream.
-
         We will call the response deferred callback passing
         the response object
+
+        event: StreamReset
+        Stream reset via RST_FRAME by the upstream hence forcefully close
+        this stream and send TODO: ?
+
+        event: None
+        No event is launched -- Hence we will simply close this stream
         """
+        # TODO: In case of abruptly stream close
+        #  Q1. Do we need to send the request again?
+        #  Q2. What response should we send now?
         assert self.stream_closed_server is False
         self.stream_closed_server = True
 
+        if not isinstance(event, StreamEnded):
+            # TODO
+            # Stream was abruptly ended here
+            # Partial - Content-Length header not provided
+            pass
+
         self._fire_response_deferred()
         self._cb_close(self.stream_id)
 
-    def _fire_response_deferred(self):
+    def _fire_response_deferred(self, flags=None):
+        """Builds response from the self._response dict
+        and fires the response deferred callback with the
+        generated response instance"""
         # TODO:
         #  1. Set flags, certificate, ip_address in response
         #  2. Should we fire this in case of
         #   2.1 StreamReset in between when data is received partially
         #   2.2 Forcefully closed the stream
+        #  3. Update Client Side Status Codes here
 
-        # NOTE: Presently on fired with successful response
-        response = Response(
+        response_cls = responsetypes.from_args(
+            headers=self._response['headers'],
             url=self._request.url,
-            status=self._response_headers[":status"],
-            headers=self._response_headers,
-            body=self._response_data,
-            request=self._request
+            body=self._response['body']
         )
+
+        # If there is :status in headers then
+        # HTTP Status Code: 499 - Client Closed Request
+        status = self._response['headers'].get(':status', '499')
+
+        response = response_cls(
+            url=self._request.url,
+            status=status,
+            headers=self._response['headers'],
+            body=self._response['body'],
+            request=self._request,
+            flags=flags,
+            certificate=self._metadata['certificate'],
+            ip_address=self._metadata['ip_address']
+        )
+
         self._deferred_response.callback(response)

From c74ef660c7b28cd69825a4b9843230a383f702f4 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sun, 21 Jun 2020 09:34:23 +0530
Subject: [PATCH 052/716] feat: handle response for different reasons

- Add StreamCloseReason enum
- Send response for different cases considering download_warnsize,
download_maxsize, fail_on_data_loss, connection lost, etc.
---
 scrapy/core/http2/protocol.py |  15 ++-
 scrapy/core/http2/stream.py   | 209 ++++++++++++++++++++++++----------
 2 files changed, 155 insertions(+), 69 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index d6134183a..188c14c15 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -11,7 +11,7 @@ from h2.events import (
 )
 from twisted.internet.protocol import connectionDone, Protocol
 
-from scrapy.core.http2.stream import Stream
+from scrapy.core.http2.stream import Stream, StreamCloseReason
 from scrapy.http import Request
 
 LOGGER = logging.getLogger(__name__)
@@ -71,7 +71,7 @@ class H2ClientProtocol(Protocol):
             stream_id=stream_id,
             request=request,
             connection=self.conn,
-            metadata=self._metadata,
+            conn_metadata=self._metadata,
             write_to_transport=self._write_to_transport,
             cb_close=self._stream_close_cb
         )
@@ -133,7 +133,7 @@ class H2ClientProtocol(Protocol):
         """
         # Pop all streams which were pending and were not yet started
         for stream_id in list(self.streams):
-            self.streams[stream_id].close()
+            self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST)
 
         self.conn.close_connection()
 
@@ -180,19 +180,18 @@ class H2ClientProtocol(Protocol):
 
     def stream_ended(self, event: StreamEnded):
         stream_id = event.stream_id
-        self.streams[stream_id].close()
+        self.streams[stream_id].close(StreamCloseReason.ENDED)
 
     def stream_reset(self, event: StreamReset):
         # TODO: event.stream_id was abruptly closed
         #  Q. What should be the response? (Failure/Partial/???)
-        self.streams[event.stream_id].close(event)
+        self.streams[event.stream_id].close(StreamCloseReason.RESET)
 
     def window_updated(self, event: WindowUpdated):
         stream_id = event.stream_id
         if stream_id != 0:
-            self.streams[stream_id].receive_window_update(event.delta)
+            self.streams[stream_id].receive_window_update()
         else:
             # Send leftover data for all the streams
             for stream in self.streams.values():
-                if stream.request_sent:
-                    stream.send_data()
+                stream.receive_window_update()
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index e8b4471d6..07a4428c8 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -1,11 +1,15 @@
 import logging
+from enum import IntFlag, auto
+from io import BytesIO
 from typing import Dict
 from urllib.parse import urlparse
 
 from h2.connection import H2Connection
-from h2.events import StreamEnded
+from h2.errors import ErrorCodes
 from h2.exceptions import StreamClosedError
-from twisted.internet.defer import Deferred
+from twisted.internet.defer import Deferred, CancelledError
+from twisted.python.failure import Failure
+from twisted.web.client import ResponseFailed
 
 from scrapy.http import Request
 from scrapy.http.headers import Headers
@@ -14,6 +18,20 @@ from scrapy.responsetypes import responsetypes
 LOGGER = logging.getLogger(__name__)
 
 
+class StreamCloseReason(IntFlag):
+    # Received a StreamEnded event
+    ENDED = auto()
+
+    # Received a StreamReset event -- ended abruptly
+    RESET = auto()
+
+    # Transport connection was lost
+    CONNECTION_LOST = auto()
+
+    # Expected response body size is more than allowed limit
+    MAXSIZE_EXCEEDED = auto()
+
+
 class Stream:
     """Represents a single HTTP/2 Stream.
 
@@ -26,13 +44,16 @@ class Stream:
     """
 
     def __init__(
-        self,
-        stream_id: int,
-        request: Request,
-        connection: H2Connection,
-        metadata: Dict,
-        write_to_transport,
-        cb_close
+            self,
+            stream_id: int,
+            request: Request,
+            connection: H2Connection,
+            conn_metadata: Dict,
+            write_to_transport,
+            cb_close,
+            download_maxsize=0,
+            download_warnsize=0,
+            fail_on_data_loss=True
     ):
         """
         Arguments:
@@ -40,7 +61,7 @@ class Stream:
                 uniquely identified by a single integer
             request {Request} -- HTTP request
             connection {H2Connection} -- HTTP/2 connection this stream belongs to.
-            metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection
+            conn_metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection
             write_to_transport {callable} -- Method used to write & send data to the server
                 This method should be used whenever some frame is to be sent to the server.
             cb_close {callable} -- Method called when this stream is closed
@@ -49,16 +70,24 @@ class Stream:
         self.stream_id = stream_id
         self._request = request
         self._conn = connection
-        self._metadata = metadata
+        self._conn_metadata = conn_metadata
         self._write_to_transport = write_to_transport
         self._cb_close = cb_close
 
-        self._request_body = self._request.body
-        self.content_length = 0 if self._request_body is None else len(self._request_body)
+        self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
+        self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
+        self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss)
+
+        self.request_start_time = None
+
+        self.content_length = 0 if self._request.body is None else len(self._request.body)
 
         # Flag to keep track whether this stream has initiated the request
         self.request_sent = False
 
+        # Flag to track whether we have logged about exceeding download warnsize
+        self._reached_warnsize = False
+
         # Each time we send a data frame, we will decrease value by the amount send.
         self.remaining_content_length = self.content_length
 
@@ -68,17 +97,17 @@ class Stream:
         # Flag to keep track whether the server has closed the stream
         self.stream_closed_server = False
 
-        # The amount of data received that counts against the flow control
-        # window
-        self._response_flow_controlled_size = 0
-
         # Private variable used to build the response
         # this response is then converted to appropriate Response class
         # passed to the response deferred callback
         self._response = {
             # Data received frame by frame from the server is appended
             # and passed to the response Deferred when completely received.
-            'body': b'',
+            'body': BytesIO(),
+
+            # The amount of data received that counts against the flow control
+            # window
+            'flow_controlled_size': 0,
 
             # Headers received after sending the request
             'headers': Headers({})
@@ -137,16 +166,8 @@ class Stream:
 
          Warning: Only call this method when stream not closed from client side
             and has initiated request already by sending HEADER frame. If not then
-            stream will be closed from client side with 499 response.
-
-            TODO: Q. Should we instead raise ProtocolError here with a proper message?
+            stream will raise ProtocolError (raise by h2 state machine).
          """
-        if self.stream_closed_local or self.stream_closed_server:
-            raise StreamClosedError(self.stream_id)
-        elif not self.request_sent:
-            self.close()
-            return
-
         # TODO:
         #  1. Add test for sending very large data
         #  2. Add test for small data
@@ -154,6 +175,9 @@ class Stream:
         #    3.1 Large number of request
         #    3.2 Small number of requests
 
+        if self.stream_closed_local:
+            raise StreamClosedError(self.stream_id)
+
         # Firstly, check what the flow control window is for current stream.
         window_size = self._conn.local_flow_control_window(stream_id=self.stream_id)
 
@@ -170,7 +194,7 @@ class Stream:
             chunk_size = min(bytes_to_send_size, max_frame_size)
 
             data_chunk_start_id = self.content_length - self.remaining_content_length
-            data_chunk = self._request_body[data_chunk_start_id:data_chunk_start_id + chunk_size]
+            data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size]
 
             self._conn.send_data(self.stream_id, data_chunk, end_stream=False)
 
@@ -183,7 +207,7 @@ class Stream:
             self,
             self.content_length - self.remaining_content_length, self.content_length,
             data_frames_sent,
-            self._metadata['ip_address'])
+            self._conn_metadata['ip_address'])
         )
 
         # End the stream if no more data needs to be send
@@ -197,24 +221,40 @@ class Stream:
         # Q. What about the rest of the data?
         # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame
 
-    def receive_window_update(self, delta):
+    def receive_window_update(self):
         """Flow control window size was changed.
         Send data that earlier could not be sent as we were
         blocked behind the flow control.
-
-        Arguments:
-            delta -- Window change delta
         """
-        if self.remaining_content_length > 0 and not self.stream_closed_server:
+        if self.remaining_content_length and not self.stream_closed_server and self.request_sent:
             self.send_data()
 
     def receive_data(self, data: bytes, flow_controlled_length: int):
-        self._response['body'] += data
-        self._response_flow_controlled_size += flow_controlled_length
+        self._response['body'].write(data)
+        self._response['flow_controlled_size'] += flow_controlled_length
+
+        if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize:
+            # Clear buffer earlier to avoid keeping data in memory for a long time
+            self._response['body'].truncate(0)
+            self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
+            return
+
+        if self._download_warnsize \
+                and self._response['flow_controlled_size'] > self._download_warnsize \
+                and not self._reached_warnsize:
+            self._reached_warnsize = True
+            warning_msg = ('Received more ({bytes}) bytes than download ',
+                           'warn size ({warnsize}) in request {request}')
+            warning_args = {
+                'bytes': self._response['flow_controlled_size'],
+                'warnsize': self._download_warnsize,
+                'request': self._request
+            }
+            LOGGER.warning(warning_msg, warning_args)
 
         # Acknowledge the data received
         self._conn.acknowledge_received_data(
-            self._response_flow_controlled_size,
+            self._response['flow_controlled_size'],
             self.stream_id
         )
 
@@ -222,35 +262,83 @@ class Stream:
         for name, value in headers:
             self._response['headers'][name] = value
 
-    def close(self, event=None):
-        """Based on the event sent we will handle each case.
+        # Check if we exceed the allowed max data size which can be received
+        expected_size = int(self._response['headers'].get(b'Content-Length', -1))
+        if self._download_maxsize and expected_size > self._download_maxsize:
+            self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
+            return
 
-        event: StreamEnded
-        Stream is ended by the server hence no further
-        data or headers should be expected on this stream.
-        We will call the response deferred callback passing
-        the response object
+        if self._download_warnsize and expected_size > self._download_warnsize:
+            warning_msg = ("Expected response size ({size}) larger than ",
+                           "download warn size ({warnsize}) in request {request}.")
+            warning_args = {
+                'size': expected_size, 'warnsize': self._download_warnsize,
+                'request': self._request
+            }
+            LOGGER.warning(warning_msg, warning_args)
 
-        event: StreamReset
-        Stream reset via RST_FRAME by the upstream hence forcefully close
-        this stream and send TODO: ?
+    def reset_stream(self, reason=StreamCloseReason.RESET):
+        """Close this stream by sending a RST_FRAME to the remote peer"""
+        # TODO: Q. REFUSED_STREAM or CANCEL ?
+        if self.stream_closed_local:
+            raise StreamClosedError(self.stream_id)
 
-        event: None
-        No event is launched -- Hence we will simply close this stream
+        self.stream_closed_local = True
+        self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
+        self._write_to_transport()
+        self.close(reason)
+
+    def _is_data_lost(self) -> bool:
+        assert self.stream_closed_server
+
+        expected_size = self._response['flow_controlled_size']
+        received_body_size = int(self._response['headers'][b'Content-Length'])
+
+        return expected_size != received_body_size
+
+    def close(self, reason: StreamCloseReason):
+        """Based on the reason sent we will handle each case.
         """
         # TODO: In case of abruptly stream close
         #  Q1. Do we need to send the request again?
         #  Q2. What response should we send now?
-        assert self.stream_closed_server is False
+        if self.stream_closed_server:
+            raise StreamClosedError(self.stream_id)
+
         self.stream_closed_server = True
 
-        if not isinstance(event, StreamEnded):
-            # TODO
-            # Stream was abruptly ended here
-            # Partial - Content-Length header not provided
-            pass
+        flags = None
+        if b'Content-Length' not in self._response['headers']:
+            # Missing Content-Length - PotentialDataLoss
+            flags = ['partial']
+        elif self._is_data_lost():
+            if self._fail_on_dataloss:
+                self._deferred_response.errback(ResponseFailed([Failure()]))
+                self._cb_close(self.stream_id)
+                return
+            else:
+                flags = ['dataloss']
+
+        if reason is StreamCloseReason.ENDED:
+            self._fire_response_deferred(flags)
+
+        elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST):
+            # Stream was abruptly ended here
+            self._deferred_response.errback(ResponseFailed([Failure()]))
+
+        elif reason is StreamCloseReason.MAXSIZE_EXCEEDED:
+            expected_size = int(self._response['headers'].get(b'Content-Length', -1))
+            error_msg = ("Cancelling download of {url}: expected response "
+                         "size ({size}) larger than download max size ({maxsize}).")
+            error_args = {
+                'url': self._request.url,
+                'size': expected_size,
+                'maxsize': self._download_maxsize
+            }
+
+            LOGGER.error(error_msg, error_args)
+            self._deferred_response.errback(CancelledError(error_msg.format(**error_args)))
 
-        self._fire_response_deferred()
         self._cb_close(self.stream_id)
 
     def _fire_response_deferred(self, flags=None):
@@ -258,7 +346,6 @@ class Stream:
         and fires the response deferred callback with the
         generated response instance"""
         # TODO:
-        #  1. Set flags, certificate, ip_address in response
         #  2. Should we fire this in case of
         #   2.1 StreamReset in between when data is received partially
         #   2.2 Forcefully closed the stream
@@ -270,7 +357,7 @@ class Stream:
             body=self._response['body']
         )
 
-        # If there is :status in headers then
+        # If there is no :status in headers then
         # HTTP Status Code: 499 - Client Closed Request
         status = self._response['headers'].get(':status', '499')
 
@@ -278,11 +365,11 @@ class Stream:
             url=self._request.url,
             status=status,
             headers=self._response['headers'],
-            body=self._response['body'],
+            body=self._response['body'].getvalue(),
             request=self._request,
             flags=flags,
-            certificate=self._metadata['certificate'],
-            ip_address=self._metadata['ip_address']
+            certificate=self._conn_metadata['certificate'],
+            ip_address=self._conn_metadata['ip_address']
         )
 
         self._deferred_response.callback(response)

From a97ac0adf86b67a16f09c41f618f736adb872503 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 24 Jun 2020 06:40:20 +0530
Subject: [PATCH 053/716] test: GET request for HTTP2Client using mockserver

---
 scrapy/core/http2/protocol.py       |  4 +-
 scrapy/core/http2/stream.py         | 10 +---
 tests/test_http2_client_protocol.py | 71 ++++++++++-------------------
 3 files changed, 30 insertions(+), 55 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 188c14c15..455d8777e 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -9,6 +9,8 @@ from h2.events import (
     DataReceived, ResponseReceived, SettingsAcknowledged,
     StreamEnded, StreamReset, WindowUpdated
 )
+
+from twisted.internet.ssl import Certificate
 from twisted.internet.protocol import connectionDone, Protocol
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
@@ -115,7 +117,7 @@ class H2ClientProtocol(Protocol):
         self.destination = self.transport.getPeer()
         LOGGER.info('Connection made to {}'.format(self.destination))
 
-        self._metadata['certificate'] = self.transport.getPeerCertificate()
+        self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
         self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host)
 
         self.conn.initiate_connection()
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 07a4428c8..112ce5bcd 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -117,7 +117,7 @@ class Stream:
         self._deferred_response = Deferred()
 
     def __str__(self):
-        return "Stream(id={})".format(self.stream_id)
+        return "Stream(id={})".format(repr(self.stream_id))
 
     __repr__ = __str__
 
@@ -299,9 +299,6 @@ class Stream:
     def close(self, reason: StreamCloseReason):
         """Based on the reason sent we will handle each case.
         """
-        # TODO: In case of abruptly stream close
-        #  Q1. Do we need to send the request again?
-        #  Q2. What response should we send now?
         if self.stream_closed_server:
             raise StreamClosedError(self.stream_id)
 
@@ -346,10 +343,7 @@ class Stream:
         and fires the response deferred callback with the
         generated response instance"""
         # TODO:
-        #  2. Should we fire this in case of
-        #   2.1 StreamReset in between when data is received partially
-        #   2.2 Forcefully closed the stream
-        #  3. Update Client Side Status Codes here
+        #  1. Update Client Side Status Codes here
 
         response_cls = responsetypes.from_args(
             headers=self._response['headers'],
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 7830f7028..a67575d3c 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -1,71 +1,50 @@
-import os
-import shutil
+from urllib.parse import urlparse
 
-from twisted.internet import defer, reactor
+from twisted.internet import reactor
 from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
-from twisted.internet.ssl import optionsForClientTLS
-from twisted.protocols.policies import WrappingFactory
-from twisted.python.filepath import FilePath
+from twisted.internet.ssl import CertificateOptions
 from twisted.trial import unittest
-from twisted.web import static, server
 
 from scrapy.core.http2.protocol import H2ClientProtocol
-from scrapy.http import Request
-from tests.mockserver import ssl_context_factory
+from scrapy.http import Request, Response
+from tests.mockserver import MockServer
 
 
 class Http2ClientProtocolTestCase(unittest.TestCase):
     scheme = 'https'
 
-    # only used for HTTPS tests
-    file_key = 'keys/localhost.key'
-    file_certificate = 'keys/localhost.crt'
-
     def setUp(self):
         # Start server for testing
-        self.path_temp = self.mktemp()
-        os.mkdir(self.path_temp)
-        FilePath(self.path_temp).child('file').setContent(b"0123456789")
-        r = static.File(self.path_temp)
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
 
-        self.site = server.Site(r, timeout=None)
-        self.wrapper = WrappingFactory(self.site)
-        self.host = 'localhost'
-        if self.scheme is 'https':
-            self.port = reactor.listenSSL(
-                0, self.wrapper,
-                ssl_context_factory(self.file_key, self.file_certificate),
-                interface=self.host
-            )
+        if self.scheme == 'https':
+            self.url = urlparse(self.mockserver.https_address)
         else:
-            self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
-
-        self.port_number = self.port.getHost().port
-
-        # Connect to the server using the custom HTTP2ClientProtocol
-        options = optionsForClientTLS(
-            hostname=self.host,
-            acceptableProtocols=[b'h2']
-        )
+            self.url = urlparse(self.mockserver.http_address)
 
         self.protocol = H2ClientProtocol()
 
-        connectProtocol(
-            endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options),
-            protocol=self.protocol
-        )
+        # Connect to the server using the custom HTTP2ClientProtocol
+        options = CertificateOptions(acceptableProtocols=[b'h2'])
+        endpoint = SSL4ClientEndpoint(reactor, self.url.hostname, self.url.port, options)
+        connectProtocol(endpoint, self.protocol)
 
     def getURL(self, path):
-        return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path)
+        return "{}://{}:{}/{}".format(self.url.scheme, self.url.hostname, self.url.port, path)
 
-    @defer.inlineCallbacks
     def tearDown(self):
-        yield self.port.stopListening()
-        shutil.rmtree(self.path_temp)
+        self.mockserver.__exit__(None, None, None)
 
     def test_download(self):
-        request = Request(self.getURL('file'))
+        request = Request(self.getURL(''))
+
+        def assert_response(response: Response):
+            self.assertEqual(response.body, b'Scrapy mock HTTP server\n')
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.request, request)
+            self.assertEqual(response.url, request.url)
+
         d = self.protocol.request(request)
-        d.addCallback(lambda response: response.body)
-        d.addCallback(self.assertEqual, b"0123456789")
+        d.addCallback(assert_response)
         return d

From 69f6d038c0bc51a9de706890621d2ce183f79e09 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 24 Jun 2020 07:06:32 +0530
Subject: [PATCH 054/716] feat: TypedDict for Stream._response
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- remove test_protocol.py as working testing environment is setup 🙂🙃
- Add typing_extensions as dependency to support TypedDict for
python<3.8
---
 scrapy/core/http2/protocol.py      | 12 +-----
 scrapy/core/http2/stream.py        | 11 ++++-
 scrapy/core/http2/test_protocol.py | 67 ------------------------------
 setup.py                           |  4 +-
 4 files changed, 15 insertions(+), 79 deletions(-)
 delete mode 100644 scrapy/core/http2/test_protocol.py

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 455d8777e..9b8ec6c77 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -9,9 +9,8 @@ from h2.events import (
     DataReceived, ResponseReceived, SettingsAcknowledged,
     StreamEnded, StreamReset, WindowUpdated
 )
-
-from twisted.internet.ssl import Certificate
 from twisted.internet.protocol import connectionDone, Protocol
+from twisted.internet.ssl import Certificate
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
 from scrapy.http import Request
@@ -22,12 +21,7 @@ LOGGER = logging.getLogger(__name__)
 class H2ClientProtocol(Protocol):
     # TODO:
     #  1. Check for user-agent while testing
-    #  2. Add support for cookies
-    #  3. Handle priority updates (Not required)
-    #  4. Handle case when received events have StreamID = 0 (applied to H2Connection)
-    #  1 & 2:
-    #   - Automatically handled by the Request middleware
-    #   - request.headers will have 'Set-Cookie' value
+    #  2. Handle case when received events have StreamID = 0 (applied to H2Connection)
 
     def __init__(self):
         config = H2Configuration(client_side=True, header_encoding='utf-8')
@@ -185,8 +179,6 @@ class H2ClientProtocol(Protocol):
         self.streams[stream_id].close(StreamCloseReason.ENDED)
 
     def stream_reset(self, event: StreamReset):
-        # TODO: event.stream_id was abruptly closed
-        #  Q. What should be the response? (Failure/Partial/???)
         self.streams[event.stream_id].close(StreamCloseReason.RESET)
 
     def window_updated(self, event: WindowUpdated):
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 112ce5bcd..023f4f4eb 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -10,11 +10,20 @@ from h2.exceptions import StreamClosedError
 from twisted.internet.defer import Deferred, CancelledError
 from twisted.python.failure import Failure
 from twisted.web.client import ResponseFailed
+# for python < 3.8 -- typing.TypedDict is undefined
+from typing_extensions import TypedDict
 
 from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
 
+
+class _ResponseTypedDict(TypedDict):
+    body: BytesIO
+    flow_controlled_size: int
+    headers: Headers
+
+
 LOGGER = logging.getLogger(__name__)
 
 
@@ -100,7 +109,7 @@ class Stream:
         # Private variable used to build the response
         # this response is then converted to appropriate Response class
         # passed to the response deferred callback
-        self._response = {
+        self._response: _ResponseTypedDict = {
             # Data received frame by frame from the server is appended
             # and passed to the response Deferred when completely received.
             'body': BytesIO(),
diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.py
deleted file mode 100644
index c7782a518..000000000
--- a/scrapy/core/http2/test_protocol.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# This is simple script to test
-
-import json
-
-from twisted.internet import reactor
-from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
-from twisted.internet.ssl import optionsForClientTLS
-
-from scrapy.core.http2.protocol import H2ClientProtocol
-from scrapy.http import Request, Response, JsonRequest
-
-try:
-    with open('data.json', 'r') as f:
-        JSON_DATA = json.load(f)
-except:
-    JSON_DATA = {
-        "data": "To test for really large amount of data -- Add data.json with lots of data.",
-        "why": "To test whether correct data is sent :)"
-    }
-
-# Use nghttp2 for testing whether basic setup works - for small response
-HTTPBIN_AUTHORITY = u'nghttp2.org'
-HTTPBIN_REQUEST_URLS = 1 * [
-    Request(url='https://nghttp2.org/httpbin/get', method='GET'),
-    Request(url='https://nghttp2.org/httpbin/post', method='POST'),
-    JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA),
-]
-
-# Use POKE_API for testing large responses
-POKE_API_AUTHORITY = u'pokeapi.co'
-POKE_API_REQUESTS = 15 * [
-    Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'),
-    Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'),
-    Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'),
-    Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'),  # should give 404
-]
-
-AUTHORITY = POKE_API_AUTHORITY
-REQUEST_URLS = POKE_API_REQUESTS
-
-options = optionsForClientTLS(
-    hostname=AUTHORITY,
-    acceptableProtocols=[b'h2'],
-)
-
-protocol = H2ClientProtocol()
-
-count_responses = 1
-
-
-def print_response(response):
-    global count_responses
-    assert isinstance(response, Response)
-    print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body)))
-    count_responses = count_responses + 1
-
-
-for request in REQUEST_URLS:
-    d = protocol.request(request)
-    d.addCallback(print_response)
-
-connectProtocol(
-    SSL4ClientEndpoint(reactor, AUTHORITY, 443, options),
-    protocol
-)
-
-reactor.run()
diff --git a/setup.py b/setup.py
index dafa5684a..d1470df5e 100644
--- a/setup.py
+++ b/setup.py
@@ -65,7 +65,7 @@ setup(
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    python_requires='>=3.5',
+    python_requires='>=3.5.2',
     install_requires=[
         'Twisted>=17.9.0',
         'Twisted[http2]>=17.9.0'
@@ -80,6 +80,8 @@ setup(
         'w3lib>=1.17.0',
         'zope.interface>=4.1.3',
         'protego>=0.1.15',
+        'itemadapter>=0.1.0',
+        'typing_extensions>=3.7'
     ],
     extras_require=extras_require,
 )

From 23da8e106822cf2805e8d74c382a31bca70986f3 Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Sat, 27 Jun 2020 20:36:45 +0530
Subject: [PATCH 055/716] Add schemaless http proxy support

---
 scrapy/core/downloader/handlers/http11.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 22c9ac520..73e56c87d 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -7,7 +7,7 @@ import warnings
 from contextlib import suppress
 from io import BytesIO
 from time import time
-from urllib.parse import urldefrag
+from urllib.parse import urldefrag, urlparse
 
 from twisted.internet import defer, protocol, ssl
 from twisted.internet.endpoints import TCP4ClientEndpoint
@@ -255,7 +255,7 @@ class ScrapyProxyAgent(Agent):
             bindAddress=bindAddress,
             pool=pool,
         )
-        self._proxyURI = URI.fromBytes(proxyURI)
+        self._proxyURI = URI.fromBytes(urlparse(proxyURI)._replace(scheme=b'http').geturl())
 
     def request(self, method, uri, headers=None, bodyProducer=None):
         """

From f53f06020b740b450c7555d562565a27918e8036 Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Sat, 27 Jun 2020 23:28:40 +0530
Subject: [PATCH 056/716] Test http schemaless proxy

---
 tests/test_downloader_handlers.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 51deb20f4..5854659dd 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -131,6 +131,7 @@ class ContentLengthHeaderResource(resource.Resource):
     A testing resource which renders itself as the value of the Content-Length
     header from the request.
     """
+
     def render(self, request):
         return request.requestHeaders.getRawHeaders(b"content-length")[0]
 
@@ -186,6 +187,7 @@ class EmptyContentTypeHeaderResource(resource.Resource):
     A testing resource which renders itself as the value of request body
     without content-type header in response.
     """
+
     def render(self, request):
         request.setHeader("content-type", "")
         return request.content.read()
@@ -733,6 +735,16 @@ class Http11ProxyTestCase(HttpProxyTestCase):
         timeout = yield self.assertFailure(d, error.TimeoutError)
         self.assertIn(domain, timeout.osError)
 
+    def test_download_with_proxy_without_http_scheme(self):
+        def _test(response):
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.url, request.url)
+            self.assertEqual(response.body, b'http://example.com')
+
+        http_proxy = self.getURL('').replace('http:', '')
+        request = Request('http://example.com', meta={'proxy': http_proxy})
+        return self.download_request(request, Spider('foo')).addCallback(_test)
+
 
 class HttpDownloadHandlerMock:
 

From 690dd7f38bfd245428bdc618dcca1fbc26284df1 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sun, 28 Jun 2020 16:35:04 +0530
Subject: [PATCH 057/716] test: GET & POST request test for h2 client

- Remove repeated dependency Twisted from setup.py
- Test for both GET & POST when
  - Only 1 request
  - Large number (=20) of requests
and
  - Small Data (10 KB) per request
  - Large Data (10 MB) per request
- Test when request is cancelled by the client'

BREAKING CHANGES
Tests raises OpenSSL.SSL.Error when run using tox. However, all tests
passes when ran using `python -m unittest`.
---
 scrapy/core/http2/protocol.py       |   4 +-
 scrapy/core/http2/stream.py         |  66 +++---
 setup.py                            |   1 -
 tests/test_http2_client_protocol.py | 321 +++++++++++++++++++++++++---
 4 files changed, 331 insertions(+), 61 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 9b8ec6c77..7fb935f10 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -129,11 +129,11 @@ class H2ClientProtocol(Protocol):
         """
         # Pop all streams which were pending and were not yet started
         for stream_id in list(self.streams):
-            self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST)
+            self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST, reason)
 
         self.conn.close_connection()
 
-        LOGGER.info("Connection lost with reason " + str(reason))
+        LOGGER.warning("Connection lost with reason " + str(reason))
 
     def _handle_events(self, events):
         """Private method which acts as a bridge between the events
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 023f4f4eb..a26a33918 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -19,8 +19,15 @@ from scrapy.responsetypes import responsetypes
 
 
 class _ResponseTypedDict(TypedDict):
+    # Data received frame by frame from the server is appended
+    # and passed to the response Deferred when completely received.
     body: BytesIO
+
+    # The amount of data received that counts against the flow control
+    # window
     flow_controlled_size: int
+
+    # Headers received after sending the request
     headers: Headers
 
 
@@ -40,6 +47,9 @@ class StreamCloseReason(IntFlag):
     # Expected response body size is more than allowed limit
     MAXSIZE_EXCEEDED = auto()
 
+    # When the response deferred is cancelled
+    CANCELLED = auto()
+
 
 class Stream:
     """Represents a single HTTP/2 Stream.
@@ -110,20 +120,16 @@ class Stream:
         # this response is then converted to appropriate Response class
         # passed to the response deferred callback
         self._response: _ResponseTypedDict = {
-            # Data received frame by frame from the server is appended
-            # and passed to the response Deferred when completely received.
             'body': BytesIO(),
-
-            # The amount of data received that counts against the flow control
-            # window
             'flow_controlled_size': 0,
-
-            # Headers received after sending the request
             'headers': Headers({})
         }
 
-        # TODO: Add canceller for the Deferred below
-        self._deferred_response = Deferred()
+        def _cancel(_):
+            # Close this stream as gracefully as possible :)
+            self.reset_stream(StreamCloseReason.CANCELLED)
+
+        self._deferred_response = Deferred(_cancel)
 
     def __str__(self):
         return "Stream(id={})".format(repr(self.stream_id))
@@ -177,13 +183,6 @@ class Stream:
             and has initiated request already by sending HEADER frame. If not then
             stream will raise ProtocolError (raise by h2 state machine).
          """
-        # TODO:
-        #  1. Add test for sending very large data
-        #  2. Add test for small data
-        #  3. Both (1) and (2) should be tested for
-        #    3.1 Large number of request
-        #    3.2 Small number of requests
-
         if self.stream_closed_local:
             raise StreamClosedError(self.stream_id)
 
@@ -221,7 +220,6 @@ class Stream:
 
         # End the stream if no more data needs to be send
         if self.remaining_content_length == 0:
-            self.stream_closed_local = True
             self._conn.end_stream(self.stream_id)
 
         # Write data to transport -- Empty the outstanding data
@@ -288,7 +286,6 @@ class Stream:
 
     def reset_stream(self, reason=StreamCloseReason.RESET):
         """Close this stream by sending a RST_FRAME to the remote peer"""
-        # TODO: Q. REFUSED_STREAM or CANCEL ?
         if self.stream_closed_local:
             raise StreamClosedError(self.stream_id)
 
@@ -305,14 +302,16 @@ class Stream:
 
         return expected_size != received_body_size
 
-    def close(self, reason: StreamCloseReason):
+    def close(self, reason: StreamCloseReason, failure=None):
         """Based on the reason sent we will handle each case.
         """
         if self.stream_closed_server:
             raise StreamClosedError(self.stream_id)
 
+        self._cb_close(self.stream_id)
         self.stream_closed_server = True
 
+        # Do nothing if the response deferred was cancelled
         flags = None
         if b'Content-Length' not in self._response['headers']:
             # Missing Content-Length - PotentialDataLoss
@@ -320,7 +319,6 @@ class Stream:
         elif self._is_data_lost():
             if self._fail_on_dataloss:
                 self._deferred_response.errback(ResponseFailed([Failure()]))
-                self._cb_close(self.stream_id)
                 return
             else:
                 flags = ['dataloss']
@@ -328,10 +326,19 @@ class Stream:
         if reason is StreamCloseReason.ENDED:
             self._fire_response_deferred(flags)
 
-        elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST):
-            # Stream was abruptly ended here
-            self._deferred_response.errback(ResponseFailed([Failure()]))
+        # Stream was abruptly ended here
+        elif reason is StreamCloseReason.CANCELLED:
+            # Client has cancelled the request. Remove all the data
+            # received and fire the response deferred with no flags set
+            self._response['body'].truncate(0)
+            self._response['headers'].clear()
+            self._fire_response_deferred()
 
+        elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST):
+            if failure is None:
+                self._deferred_response.errback(ResponseFailed([Failure()]))
+            else:
+                self._deferred_response.errback(failure)
         elif reason is StreamCloseReason.MAXSIZE_EXCEEDED:
             expected_size = int(self._response['headers'].get(b'Content-Length', -1))
             error_msg = ("Cancelling download of {url}: expected response "
@@ -345,22 +352,21 @@ class Stream:
             LOGGER.error(error_msg, error_args)
             self._deferred_response.errback(CancelledError(error_msg.format(**error_args)))
 
-        self._cb_close(self.stream_id)
-
     def _fire_response_deferred(self, flags=None):
         """Builds response from the self._response dict
         and fires the response deferred callback with the
         generated response instance"""
-        # TODO:
-        #  1. Update Client Side Status Codes here
+        # TODO: Update Client Side Status Codes here
 
+        body = self._response['body'].getvalue()
         response_cls = responsetypes.from_args(
             headers=self._response['headers'],
             url=self._request.url,
-            body=self._response['body']
+            body=body
         )
 
-        # If there is no :status in headers then
+        # If there is no :status in headers
+        # (happens when client called response_deferred.cancel())
         # HTTP Status Code: 499 - Client Closed Request
         status = self._response['headers'].get(':status', '499')
 
@@ -368,7 +374,7 @@ class Stream:
             url=self._request.url,
             status=status,
             headers=self._response['headers'],
-            body=self._response['body'].getvalue(),
+            body=body,
             request=self._request,
             flags=flags,
             certificate=self._conn_metadata['certificate'],
diff --git a/setup.py b/setup.py
index d1470df5e..575c74e7f 100644
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,6 @@ setup(
     ],
     python_requires='>=3.5.2',
     install_requires=[
-        'Twisted>=17.9.0',
         'Twisted[http2]>=17.9.0'
         'cryptography>=2.0',
         'cssselect>=0.9.1',
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index a67575d3c..0f3730c9e 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -1,50 +1,315 @@
-from urllib.parse import urlparse
+# TODO: Add test cases for
+#   1. No Content Length response header
+#   2. Cancel Response Deferred
+import json
+import os
+import random
+import shutil
+import string
 
 from twisted.internet import reactor
-from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint
-from twisted.internet.ssl import CertificateOptions
-from twisted.trial import unittest
+from twisted.internet.defer import inlineCallbacks, DeferredList
+from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint
+from twisted.internet.protocol import Factory
+from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate
+from twisted.trial.unittest import TestCase
+from twisted.web.http import Request as TxRequest
+from twisted.web.resource import Resource
+from twisted.web.server import Site
+from twisted.web.static import File
 
 from scrapy.core.http2.protocol import H2ClientProtocol
-from scrapy.http import Request, Response
-from tests.mockserver import MockServer
+from scrapy.http import Request, Response, JsonRequest
+from tests.mockserver import ssl_context_factory
 
 
-class Http2ClientProtocolTestCase(unittest.TestCase):
+def generate_random_string(size):
+    return ''.join(random.choices(
+        string.ascii_uppercase + string.digits,
+        k=size
+    ))
+
+
+def make_html_body(val):
+    response = '''<html>
+<h1>Hello from HTTP2<h1>
+<p>{}</p>
+</html>'''.format(val)
+    return bytes(response, 'utf-8')
+
+
+class Data:
+    SMALL_SIZE = 1024 * 10  # 10 KB
+    LARGE_SIZE = (1024 ** 2) * 10  # 10 MB
+
+    STR_SMALL = generate_random_string(SMALL_SIZE)
+    STR_LARGE = generate_random_string(LARGE_SIZE)
+
+    EXTRA_SMALL = generate_random_string(1024 * 15)
+    EXTRA_LARGE = generate_random_string((1024 ** 2) * 15)
+
+    HTML_SMALL = make_html_body(STR_SMALL)
+    HTML_LARGE = make_html_body(STR_LARGE)
+
+    JSON_SMALL = {'data': STR_SMALL}
+    JSON_LARGE = {'data': STR_LARGE}
+
+
+class LeafResource(Resource):
+    isLeaf = True
+
+
+class GetDataHtmlSmall(LeafResource):
+    def render_GET(self, request: TxRequest):
+        request.setHeader('Content-Type', 'text/html; charset=UTF-8')
+        return Data.HTML_SMALL
+
+
+class GetDataHtmlLarge(LeafResource):
+    def render_GET(self, request: TxRequest):
+        request.setHeader('Content-Type', 'text/html; charset=UTF-8')
+        return Data.HTML_LARGE
+
+
+class PostDataJsonMixin:
+    @staticmethod
+    def make_response(request: TxRequest, extra_data: str):
+        response = {
+            'request-headers': {},
+            'request-body': json.loads(request.content.read()),
+            'extra-data': extra_data
+        }
+        for k, v in request.requestHeaders.getAllRawHeaders():
+            response['request-headers'][k.decode('utf-8')] = v[0].decode('utf-8')
+
+        response_bytes = bytes(json.dumps(response), 'utf-8')
+        request.setHeader('Content-Type', 'application/json')
+        return response_bytes
+
+
+class PostDataJsonSmall(LeafResource, PostDataJsonMixin):
+    def render_POST(self, request: TxRequest):
+        return self.make_response(request, Data.EXTRA_SMALL)
+
+
+class PostDataJsonLarge(LeafResource, PostDataJsonMixin):
+    def render_POST(self, request: TxRequest):
+        return self.make_response(request, Data.EXTRA_LARGE)
+
+
+def get_client_certificate(key_file, certificate_file):
+    with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate:
+        pem = ''.join(key.readlines()) + ''.join(certificate.readlines())
+
+    return PrivateCertificate.loadPEM(pem)
+
+
+class Https2ClientProtocolTestCase(TestCase):
     scheme = 'https'
+    key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key')
+    certificate_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.crt')
 
+    def _init_resource(self):
+        self.temp_directory = self.mktemp()
+        os.mkdir(self.temp_directory)
+        r = File(self.temp_directory)
+        r.putChild(b'get-data-html-small', GetDataHtmlSmall())
+        r.putChild(b'get-data-html-large', GetDataHtmlLarge())
+
+        r.putChild(b'post-data-json-small', PostDataJsonSmall())
+        r.putChild(b'post-data-json-large', PostDataJsonLarge())
+        return r
+
+    @inlineCallbacks
     def setUp(self):
+        # Initialize resource tree
+        root = self._init_resource()
+        self.site = Site(root, timeout=None)
+
         # Start server for testing
-        self.mockserver = MockServer()
-        self.mockserver.__enter__()
-
+        self.hostname = u'localhost'
         if self.scheme == 'https':
-            self.url = urlparse(self.mockserver.https_address)
+            context_factory = ssl_context_factory(self.key_file, self.certificate_file)
+            server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname)
         else:
-            self.url = urlparse(self.mockserver.http_address)
+            server_endpoint = TCP4ServerEndpoint(reactor, 0, interface=self.hostname)
+        self.server = yield server_endpoint.listen(self.site)
+        self.port_number = self.server.getHost().port
 
-        self.protocol = H2ClientProtocol()
-
-        # Connect to the server using the custom HTTP2ClientProtocol
-        options = CertificateOptions(acceptableProtocols=[b'h2'])
-        endpoint = SSL4ClientEndpoint(reactor, self.url.hostname, self.url.port, options)
-        connectProtocol(endpoint, self.protocol)
-
-    def getURL(self, path):
-        return "{}://{}:{}/{}".format(self.url.scheme, self.url.hostname, self.url.port, path)
+        # Connect H2 client with server
+        client_certificate = get_client_certificate(self.key_file, self.certificate_file)
+        client_options = optionsForClientTLS(
+            hostname=self.hostname,
+            trustRoot=client_certificate,
+            acceptableProtocols=[b'h2']
+        )
+        h2_client_factory = Factory.forProtocol(H2ClientProtocol)
+        client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
+        self.client = yield client_endpoint.connect(h2_client_factory)
 
+    @inlineCallbacks
     def tearDown(self):
-        self.mockserver.__exit__(None, None, None)
+        yield self.client.transport.loseConnection()
+        yield self.client.transport.abortConnection()
+        yield self.server.stopListening()
+        shutil.rmtree(self.temp_directory)
 
-    def test_download(self):
-        request = Request(self.getURL(''))
+    def get_url(self, path):
+        """
+        :param path: Should have / at the starting compulsorily if not empty
+        :return: Complete url
+        """
+        assert len(path) > 0 and (path[0] == '/' or path[0] == '&')
+        return "{}://{}:{}{}".format(self.scheme, self.hostname, self.port_number, path)
 
-        def assert_response(response: Response):
-            self.assertEqual(response.body, b'Scrapy mock HTTP server\n')
-            self.assertEqual(response.status, 200)
+    @staticmethod
+    def _check_repeat(get_deferred, count):
+        d_list = []
+        for _ in range(count):
+            d = get_deferred()
+            d_list.append(d)
+
+        return DeferredList(d_list, fireOnOneErrback=True)
+
+    def _check_GET(
+            self,
+            request: Request,
+            expected_body,
+            expected_status
+    ):
+        def check_response(response: Response):
+            self.assertEqual(response.status, expected_status)
+            self.assertEqual(response.body, expected_body)
             self.assertEqual(response.request, request)
             self.assertEqual(response.url, request.url)
 
-        d = self.protocol.request(request)
+            content_length = int(response.headers.get('Content-Length'))
+            self.assertEqual(len(response.body), content_length)
+
+        d = self.client.request(request)
+        d.addCallback(check_response)
+        d.addErrback(self.fail)
+        return d
+
+    def test_GET_small_body(self):
+        request = Request(self.get_url('/get-data-html-small'))
+        return self._check_GET(request, Data.HTML_SMALL, 200)
+
+    def test_GET_large_body(self):
+        request = Request(self.get_url('/get-data-html-large'))
+        return self._check_GET(request, Data.HTML_LARGE, 200)
+
+    def _check_GET_x20(self, *args, **kwargs):
+        def get_deferred():
+            return self._check_GET(*args, **kwargs)
+
+        return self._check_repeat(get_deferred, 20)
+
+    def test_GET_small_body_x20(self):
+        return self._check_GET_x20(
+            Request(self.get_url('/get-data-html-small')),
+            Data.HTML_SMALL,
+            200
+        )
+
+    def test_GET_large_body_x20(self):
+        return self._check_GET_x20(
+            Request(self.get_url('/get-data-html-large')),
+            Data.HTML_LARGE,
+            200
+        )
+
+    def _check_POST_json(
+            self,
+            request: Request,
+            expected_request_body,
+            expected_extra_data,
+            expected_status: int
+    ):
+        d = self.client.request(request)
+
+        def assert_response(response: Response):
+            self.assertEqual(response.status, expected_status)
+            self.assertEqual(response.request, request)
+            self.assertEqual(response.url, request.url)
+
+            content_length = int(response.headers.get('Content-Length'))
+            self.assertEqual(len(response.body), content_length)
+
+            # Parse the body
+            body = json.loads(response.body.decode('utf-8'))
+            self.assertIn('request-body', body)
+            self.assertIn('extra-data', body)
+            self.assertIn('request-headers', body)
+
+            request_body = body['request-body']
+            self.assertEqual(request_body, expected_request_body)
+
+            extra_data = body['extra-data']
+            self.assertEqual(extra_data, expected_extra_data)
+
+            # Check if headers were sent successfully
+            request_headers = body['request-headers']
+            for k, v in request.headers.items():
+                k_str = k.decode('utf-8')
+                self.assertIn(k_str, request_headers)
+                self.assertEqual(request_headers[k_str], v[0].decode('utf-8'))
+
         d.addCallback(assert_response)
         return d
+
+    def test_POST_small_json(self):
+        request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL)
+        return self._check_POST_json(
+            request,
+            Data.JSON_SMALL,
+            Data.EXTRA_SMALL,
+            200
+        )
+
+    def test_POST_large_json(self):
+        request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE)
+        return self._check_POST_json(
+            request,
+            Data.JSON_LARGE,
+            Data.EXTRA_LARGE,
+            200
+        )
+
+    def _check_POST_json_x20(self, *args, **kwargs):
+        def get_deferred():
+            return self._check_POST_json(*args, **kwargs)
+
+        return self._check_repeat(get_deferred, 20)
+
+    def test_POST_small_json_x20(self):
+        request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL)
+        return self._check_POST_json_x20(
+            request,
+            Data.JSON_SMALL,
+            Data.EXTRA_SMALL,
+            200
+        )
+
+    def test_POST_large_json_x20(self):
+        request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE)
+        return self._check_POST_json_x20(
+            request,
+            Data.JSON_LARGE,
+            Data.EXTRA_LARGE,
+            200
+        )
+
+    def test_cancel_request(self):
+        request = Request(url=self.get_url('/get-data-html-large'))
+
+        def assert_response(response: Response):
+            self.assertEqual(response.status, 499)
+            self.assertEqual(response.request, request)
+            self.assertEqual(response.url, request.url)
+
+        d = self.client.request(request)
+        d.addCallback(assert_response)
+        d.cancel()
+
+        return d

From 6387445ef519124f393a20657e66383340fb1677 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sun, 28 Jun 2020 18:44:57 +0530
Subject: [PATCH 058/716] test(tox.ini): change Twisted -> Twisted[http2]

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 27d21ade2..ada211b3c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -77,7 +77,7 @@ deps =
     pyOpenSSL==16.2.0
     queuelib==1.4.2
     service_identity==16.0.0
-    Twisted==17.9.0
+    Twisted[http2]==17.9.0
     w3lib==1.17.0
     zope.interface==4.1.3
     -rtests/requirements-py3.txt

From 23906b6bee953d9bc5dd8042e785711b11840797 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Mon, 29 Jun 2020 18:21:05 +0530
Subject: [PATCH 059/716] refactor: move TypedDict types to types.py

- rename LOGGER -> logger
- remove self._write_to_transport from Stream class and handle all
transport related activities inside HTTP2ClientProtocol class
---
 scrapy/core/http2/protocol.py |  69 +++++++-----
 scrapy/core/http2/stream.py   | 206 +++++++++++++++++-----------------
 scrapy/core/http2/types.py    |  30 +++++
 setup.py                      |   2 +-
 4 files changed, 174 insertions(+), 133 deletions(-)
 create mode 100644 scrapy/core/http2/types.py

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 7fb935f10..0b3e5d304 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -2,6 +2,7 @@ import ipaddress
 import itertools
 import logging
 from collections import deque
+from typing import Union, Dict
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
@@ -9,20 +10,18 @@ from h2.events import (
     DataReceived, ResponseReceived, SettingsAcknowledged,
     StreamEnded, StreamReset, WindowUpdated
 )
+from h2.exceptions import ProtocolError
 from twisted.internet.protocol import connectionDone, Protocol
 from twisted.internet.ssl import Certificate
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
+from scrapy.core.http2.types import H2ConnectionMetadataDict
 from scrapy.http import Request
 
-LOGGER = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
 
 
 class H2ClientProtocol(Protocol):
-    # TODO:
-    #  1. Check for user-agent while testing
-    #  2. Handle case when received events have StreamID = 0 (applied to H2Connection)
-
     def __init__(self):
         config = H2Configuration(client_side=True, header_encoding='utf-8')
         self.conn = H2Connection(config=config)
@@ -37,7 +36,7 @@ class H2ClientProtocol(Protocol):
         self.stream_id_count = itertools.count(start=1, step=2)
 
         # Streams are stored in a dictionary keyed off their stream IDs
-        self.streams = {}
+        self.streams: Dict[int, Stream] = {}
 
         # Boolean to keep track the connection is made
         # If requests are received before connection is made
@@ -46,9 +45,11 @@ class H2ClientProtocol(Protocol):
         self.is_connection_made = False
         self._pending_request_stream_pool = deque()
 
-        # Some meta data of this connection
-        # initialized when connection is successfully made
-        self._metadata = {
+        # Save an instance of ProtocolError raised by hyper-h2
+        # We pass this instance to the streams ResponseFailed() failure
+        self._protocol_error: Union[None, ProtocolError] = None
+
+        self._metadata: H2ConnectionMetadataDict = {
             'certificate': None,
             'ip_address': None
         }
@@ -68,7 +69,6 @@ class H2ClientProtocol(Protocol):
             request=request,
             connection=self.conn,
             conn_metadata=self._metadata,
-            write_to_transport=self._write_to_transport,
             cb_close=self._stream_close_cb
         )
 
@@ -89,10 +89,10 @@ class H2ClientProtocol(Protocol):
         data = self.conn.data_to_send()
         self.transport.write(data)
 
-        LOGGER.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address']))
+        logger.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address']))
 
-    def request(self, _request: Request):
-        stream = self._new_stream(_request)
+    def request(self, request: Request):
+        stream = self._new_stream(request)
         d = stream.get_response()
 
         # If connection is not yet established then add the
@@ -109,7 +109,7 @@ class H2ClientProtocol(Protocol):
         sending some data now: we should open with the connection preamble.
         """
         self.destination = self.transport.getPeer()
-        LOGGER.info('Connection made to {}'.format(self.destination))
+        logger.info('Connection made to {}'.format(self.destination))
 
         self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
         self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host)
@@ -119,21 +119,37 @@ class H2ClientProtocol(Protocol):
         self.is_connection_made = True
 
     def dataReceived(self, data):
-        events = self.conn.receive_data(data)
-        self._handle_events(events)
-        self._write_to_transport()
+        try:
+            events = self.conn.receive_data(data)
+            self._handle_events(events)
+        except ProtocolError as e:
+            # TODO: In case of InvalidBodyLengthError -- terminate only one stream
+
+            # Save this error as ultimately the connection will be dropped
+            # internally by hyper-h2. Saved error will be passed to all the streams
+            # closed with the connection.
+            self._protocol_error = e
+
+            # We lose the transport connection here
+            self.transport.loseConnection()
+        finally:
+            self._write_to_transport()
 
     def connectionLost(self, reason=connectionDone):
         """Called by Twisted when the transport connection is lost.
         No need to write anything to transport here.
         """
         # Pop all streams which were pending and were not yet started
-        for stream_id in list(self.streams):
-            self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST, reason)
+        # NOTE: Stream.close() pops the element from the streams dictionary
+        # which raises `RuntimeError: dictionary changed size during iteration`
+        # Hence, we copy the streams into a list.
+        for stream in list(self.streams.values()):
+            stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error)
 
         self.conn.close_connection()
 
-        LOGGER.warning("Connection lost with reason " + str(reason))
+        if not reason.check(connectionDone):
+            logger.warning("Connection lost with reason " + str(reason))
 
     def _handle_events(self, events):
         """Private method which acts as a bridge between the events
@@ -144,7 +160,7 @@ class H2ClientProtocol(Protocol):
                 triggered by sending data
         """
         for event in events:
-            LOGGER.debug(event)
+            logger.debug(event)
             if isinstance(event, DataReceived):
                 self.data_received(event)
             elif isinstance(event, ResponseReceived):
@@ -158,16 +174,14 @@ class H2ClientProtocol(Protocol):
             elif isinstance(event, SettingsAcknowledged):
                 self.settings_acknowledged(event)
             else:
-                LOGGER.info("Received unhandled event {}".format(event))
+                logger.info("Received unhandled event {}".format(event))
 
     # Event handler functions starts here
     def data_received(self, event: DataReceived):
-        stream_id = event.stream_id
-        self.streams[stream_id].receive_data(event.data, event.flow_controlled_length)
+        self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length)
 
     def response_received(self, event: ResponseReceived):
-        stream_id = event.stream_id
-        self.streams[stream_id].receive_headers(event.headers)
+        self.streams[event.stream_id].receive_headers(event.headers)
 
     def settings_acknowledged(self, event: SettingsAcknowledged):
         # Send off all the pending requests
@@ -175,8 +189,7 @@ class H2ClientProtocol(Protocol):
         self._send_pending_requests()
 
     def stream_ended(self, event: StreamEnded):
-        stream_id = event.stream_id
-        self.streams[stream_id].close(StreamCloseReason.ENDED)
+        self.streams[event.stream_id].close(StreamCloseReason.ENDED)
 
     def stream_reset(self, event: StreamReset):
         self.streams[event.stream_id].close(StreamCloseReason.RESET)
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index a26a33918..da0181d52 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -1,7 +1,7 @@
 import logging
-from enum import IntFlag, auto
+from enum import Enum
 from io import BytesIO
-from typing import Dict
+from typing import Callable, List
 from urllib.parse import urlparse
 
 from h2.connection import H2Connection
@@ -10,45 +10,31 @@ from h2.exceptions import StreamClosedError
 from twisted.internet.defer import Deferred, CancelledError
 from twisted.python.failure import Failure
 from twisted.web.client import ResponseFailed
-# for python < 3.8 -- typing.TypedDict is undefined
-from typing_extensions import TypedDict
 
+from scrapy.core.http2.types import H2ConnectionMetadataDict, H2ResponseDict
 from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
 
-
-class _ResponseTypedDict(TypedDict):
-    # Data received frame by frame from the server is appended
-    # and passed to the response Deferred when completely received.
-    body: BytesIO
-
-    # The amount of data received that counts against the flow control
-    # window
-    flow_controlled_size: int
-
-    # Headers received after sending the request
-    headers: Headers
+logger = logging.getLogger(__name__)
 
 
-LOGGER = logging.getLogger(__name__)
-
-
-class StreamCloseReason(IntFlag):
+class StreamCloseReason(Enum):
     # Received a StreamEnded event
-    ENDED = auto()
+    ENDED = 1
 
     # Received a StreamReset event -- ended abruptly
-    RESET = auto()
+    RESET = 2
 
     # Transport connection was lost
-    CONNECTION_LOST = auto()
+    CONNECTION_LOST = 3
 
     # Expected response body size is more than allowed limit
-    MAXSIZE_EXCEEDED = auto()
+    MAXSIZE_EXCEEDED = 4
 
-    # When the response deferred is cancelled
-    CANCELLED = auto()
+    # When the response deferred is cancelled by the client
+    # (happens when client called response_deferred.cancel())
+    CANCELLED = 5
 
 
 class Stream:
@@ -63,34 +49,30 @@ class Stream:
     """
 
     def __init__(
-            self,
-            stream_id: int,
-            request: Request,
-            connection: H2Connection,
-            conn_metadata: Dict,
-            write_to_transport,
-            cb_close,
-            download_maxsize=0,
-            download_warnsize=0,
-            fail_on_data_loss=True
+        self,
+        stream_id: int,
+        request: Request,
+        connection: H2Connection,
+        conn_metadata: H2ConnectionMetadataDict,
+        cb_close: Callable[[int], None],
+        download_maxsize: int = 0,
+        download_warnsize: int = 0,
+        fail_on_data_loss: bool = True
     ):
         """
         Arguments:
-            stream_id {int} -- For one HTTP/2 connection each stream is
+            stream_id -- For one HTTP/2 connection each stream is
                 uniquely identified by a single integer
-            request {Request} -- HTTP request
-            connection {H2Connection} -- HTTP/2 connection this stream belongs to.
-            conn_metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection
-            write_to_transport {callable} -- Method used to write & send data to the server
-                This method should be used whenever some frame is to be sent to the server.
-            cb_close {callable} -- Method called when this stream is closed
+            request -- HTTP request
+            connection -- HTTP/2 connection this stream belongs to.
+            conn_metadata -- Reference to dictionary having metadata of HTTP/2 connection
+            cb_close -- Method called when this stream is closed
                 to notify the TCP connection instance.
         """
         self.stream_id = stream_id
         self._request = request
         self._conn = connection
         self._conn_metadata = conn_metadata
-        self._write_to_transport = write_to_transport
         self._cb_close = cb_close
 
         self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
@@ -119,7 +101,7 @@ class Stream:
         # Private variable used to build the response
         # this response is then converted to appropriate Response class
         # passed to the response deferred callback
-        self._response: _ResponseTypedDict = {
+        self._response: H2ResponseDict = {
             'body': BytesIO(),
             'flow_controlled_size': 0,
             'headers': Headers({})
@@ -136,6 +118,25 @@ class Stream:
 
     __repr__ = __str__
 
+    @property
+    def _log_warnsize(self) -> bool:
+        """Checks if we have received data which exceeds the download warnsize
+        and whether we have not already logged about it.
+
+        Returns:
+            True if both the above conditions hold true
+            False if any of the conditions is false
+        """
+        content_length_header = int(self._response['headers'].get(b'Content-Length', -1))
+        return (
+            self._download_warnsize
+            and (
+                self._response['flow_controlled_size'] > self._download_warnsize
+                or content_length_header > self._download_warnsize
+            )
+            and not self._reached_warnsize
+        )
+
     def get_response(self):
         """Simply return a Deferred which fires when response
         from the asynchronous request is available
@@ -166,10 +167,7 @@ class Stream:
     def initiate_request(self):
         headers = self._get_request_headers()
         self._conn.send_headers(self.stream_id, headers, end_stream=False)
-        self._write_to_transport()
-
         self.request_sent = True
-
         self.send_data()
 
     def send_data(self):
@@ -211,20 +209,19 @@ class Stream:
             self.remaining_content_length = self.remaining_content_length - chunk_size
 
         self.remaining_content_length = max(0, self.remaining_content_length)
-        LOGGER.debug("{} sending {}/{} data bytes ({} frames) to {}".format(
-            self,
-            self.content_length - self.remaining_content_length, self.content_length,
-            data_frames_sent,
-            self._conn_metadata['ip_address'])
+        logger.debug(
+            "{stream} sending {received}/{expected} data bytes ({frames} frames) to {ip_address}".format(
+                stream=self,
+                received=self.content_length - self.remaining_content_length,
+                expected=self.content_length,
+                frames=data_frames_sent,
+                ip_address=self._conn_metadata['ip_address'])
         )
 
         # End the stream if no more data needs to be send
         if self.remaining_content_length == 0:
             self._conn.end_stream(self.stream_id)
 
-        # Write data to transport -- Empty the outstanding data
-        self._write_to_transport()
-
         # Q. What about the rest of the data?
         # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame
 
@@ -240,24 +237,21 @@ class Stream:
         self._response['body'].write(data)
         self._response['flow_controlled_size'] += flow_controlled_length
 
+        # We check maxsize here in case the Content-Length header was not received
         if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize:
-            # Clear buffer earlier to avoid keeping data in memory for a long time
-            self._response['body'].truncate(0)
             self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
             return
 
-        if self._download_warnsize \
-                and self._response['flow_controlled_size'] > self._download_warnsize \
-                and not self._reached_warnsize:
+        if self._log_warnsize:
             self._reached_warnsize = True
-            warning_msg = ('Received more ({bytes}) bytes than download ',
-                           'warn size ({warnsize}) in request {request}')
+            warning_msg = 'Received more ({bytes}) bytes than download ' \
+                          + 'warn size ({warnsize}) in request {request}'
             warning_args = {
                 'bytes': self._response['flow_controlled_size'],
                 'warnsize': self._download_warnsize,
                 'request': self._request
             }
-            LOGGER.warning(warning_msg, warning_args)
+            logger.warning(warning_msg.format(**warning_args))
 
         # Acknowledge the data received
         self._conn.acknowledge_received_data(
@@ -275,23 +269,27 @@ class Stream:
             self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
             return
 
-        if self._download_warnsize and expected_size > self._download_warnsize:
-            warning_msg = ("Expected response size ({size}) larger than ",
-                           "download warn size ({warnsize}) in request {request}.")
+        if self._log_warnsize:
+            self._reached_warnsize = True
+            warning_msg = 'Expected response size ({size}) larger than ' \
+                          + 'download warn size ({warnsize}) in request {request}'
             warning_args = {
-                'size': expected_size, 'warnsize': self._download_warnsize,
+                'size': expected_size,
+                'warnsize': self._download_warnsize,
                 'request': self._request
             }
-            LOGGER.warning(warning_msg, warning_args)
+            logger.warning(warning_msg.format(**warning_args))
 
     def reset_stream(self, reason=StreamCloseReason.RESET):
         """Close this stream by sending a RST_FRAME to the remote peer"""
         if self.stream_closed_local:
             raise StreamClosedError(self.stream_id)
 
+        # Clear buffer earlier to avoid keeping data in memory for a long time
+        self._response['body'].truncate(0)
+
         self.stream_closed_local = True
         self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
-        self._write_to_transport()
         self.close(reason)
 
     def _is_data_lost(self) -> bool:
@@ -302,8 +300,11 @@ class Stream:
 
         return expected_size != received_body_size
 
-    def close(self, reason: StreamCloseReason, failure=None):
+    def close(self, reason: StreamCloseReason, error: Exception = None):
         """Based on the reason sent we will handle each case.
+
+        Arguments:
+            reason -- One if StreamCloseReason
         """
         if self.stream_closed_server:
             raise StreamClosedError(self.stream_id)
@@ -311,35 +312,16 @@ class Stream:
         self._cb_close(self.stream_id)
         self.stream_closed_server = True
 
-        # Do nothing if the response deferred was cancelled
         flags = None
         if b'Content-Length' not in self._response['headers']:
-            # Missing Content-Length - PotentialDataLoss
+            # Missing Content-Length - {twisted.web.http.PotentialDataLoss}
             flags = ['partial']
-        elif self._is_data_lost():
-            if self._fail_on_dataloss:
-                self._deferred_response.errback(ResponseFailed([Failure()]))
-                return
-            else:
-                flags = ['dataloss']
 
-        if reason is StreamCloseReason.ENDED:
-            self._fire_response_deferred(flags)
-
-        # Stream was abruptly ended here
-        elif reason is StreamCloseReason.CANCELLED:
-            # Client has cancelled the request. Remove all the data
-            # received and fire the response deferred with no flags set
-            self._response['body'].truncate(0)
-            self._response['headers'].clear()
-            self._fire_response_deferred()
-
-        elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST):
-            if failure is None:
-                self._deferred_response.errback(ResponseFailed([Failure()]))
-            else:
-                self._deferred_response.errback(failure)
-        elif reason is StreamCloseReason.MAXSIZE_EXCEEDED:
+        # NOTE: Order of handling the events is important here
+        # As we immediately cancel the request when maxsize is exceeded while
+        # receiving DATA_FRAME's when we have received the headers (not
+        # having Content-Length)
+        if reason is StreamCloseReason.MAXSIZE_EXCEEDED:
             expected_size = int(self._response['headers'].get(b'Content-Length', -1))
             error_msg = ("Cancelling download of {url}: expected response "
                          "size ({size}) larger than download max size ({maxsize}).")
@@ -349,14 +331,34 @@ class Stream:
                 'maxsize': self._download_maxsize
             }
 
-            LOGGER.error(error_msg, error_args)
+            logger.error(error_msg, error_args)
             self._deferred_response.errback(CancelledError(error_msg.format(**error_args)))
 
-    def _fire_response_deferred(self, flags=None):
+        elif reason is StreamCloseReason.ENDED:
+            self._fire_response_deferred(flags)
+
+        # Stream was abruptly ended here
+        elif reason is StreamCloseReason.CANCELLED:
+            # Client has cancelled the request. Remove all the data
+            # received and fire the response deferred with no flags set
+
+            # NOTE: The data is already flushed in Stream.reset_stream() called
+            # immediately when the stream needs to be cancelled
+
+            # There maybe no :status in headers, we make
+            # HTTP Status Code: 499 - Client Closed Request
+            self._response['headers'][':status'] = '499'
+            self._fire_response_deferred()
+
+        elif reason in (StreamCloseReason.RESET, StreamCloseReason.CONNECTION_LOST):
+            self._deferred_response.errback(ResponseFailed([
+                error if error else Failure()
+            ]))
+
+    def _fire_response_deferred(self, flags: List[str] = None):
         """Builds response from the self._response dict
         and fires the response deferred callback with the
         generated response instance"""
-        # TODO: Update Client Side Status Codes here
 
         body = self._response['body'].getvalue()
         response_cls = responsetypes.from_args(
@@ -365,11 +367,7 @@ class Stream:
             body=body
         )
 
-        # If there is no :status in headers
-        # (happens when client called response_deferred.cancel())
-        # HTTP Status Code: 499 - Client Closed Request
-        status = self._response['headers'].get(':status', '499')
-
+        status = self._response['headers'][':status']
         response = response_cls(
             url=self._request.url,
             status=status,
diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py
new file mode 100644
index 000000000..f28bf9472
--- /dev/null
+++ b/scrapy/core/http2/types.py
@@ -0,0 +1,30 @@
+from io import BytesIO
+from ipaddress import IPv4Address, IPv6Address
+from typing import Union
+
+from twisted.internet.ssl import Certificate
+# for python < 3.8 -- typing.TypedDict is undefined
+from typing_extensions import TypedDict
+
+from scrapy.http.headers import Headers
+
+
+class H2ConnectionMetadataDict(TypedDict):
+    """Some meta data of this connection
+    initialized when connection is successfully made
+    """
+    certificate: Union[None, Certificate]
+    ip_address: Union[None, IPv4Address, IPv6Address]
+
+
+class H2ResponseDict(TypedDict):
+    # Data received frame by frame from the server is appended
+    # and passed to the response Deferred when completely received.
+    body: BytesIO
+
+    # The amount of data received that counts against the flow control
+    # window
+    flow_controlled_size: int
+
+    # Headers received after sending the request
+    headers: Headers
diff --git a/setup.py b/setup.py
index 575c74e7f..8e50733e6 100644
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,7 @@ setup(
     ],
     python_requires='>=3.5.2',
     install_requires=[
-        'Twisted[http2]>=17.9.0'
+        'Twisted[http2]>=17.9.0',
         'cryptography>=2.0',
         'cssselect>=0.9.1',
         'lxml>=3.5.0',

From 90a7007f8818dbc224dbe51b95f07199cb730204 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Mon, 29 Jun 2020 18:29:31 +0530
Subject: [PATCH 060/716] test: warnsize logs, no content header, dataloss

---
 tests/test_http2_client_protocol.py | 167 +++++++++++++++++++++++-----
 1 file changed, 142 insertions(+), 25 deletions(-)

diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 0f3730c9e..0a2719d23 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -1,26 +1,26 @@
-# TODO: Add test cases for
-#   1. No Content Length response header
-#   2. Cancel Response Deferred
 import json
 import os
 import random
+import re
 import shutil
 import string
 
+from h2.exceptions import InvalidBodyLengthError
 from twisted.internet import reactor
-from twisted.internet.defer import inlineCallbacks, DeferredList
+from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint
 from twisted.internet.protocol import Factory
 from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate
+from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
 from twisted.web.http import Request as TxRequest
-from twisted.web.resource import Resource
-from twisted.web.server import Site
+from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.static import File
 
 from scrapy.core.http2.protocol import H2ClientProtocol
 from scrapy.http import Request, Response, JsonRequest
-from tests.mockserver import ssl_context_factory
+from scrapy.utils.python import to_bytes, to_unicode
+from tests.mockserver import ssl_context_factory, LeafResource
 
 
 def generate_random_string(size):
@@ -35,7 +35,7 @@ def make_html_body(val):
 <h1>Hello from HTTP2<h1>
 <p>{}</p>
 </html>'''.format(val)
-    return bytes(response, 'utf-8')
+    return to_bytes(response)
 
 
 class Data:
@@ -54,9 +54,8 @@ class Data:
     JSON_SMALL = {'data': STR_SMALL}
     JSON_LARGE = {'data': STR_LARGE}
 
-
-class LeafResource(Resource):
-    isLeaf = True
+    DATALOSS = b'Dataloss Content'
+    NO_CONTENT_LENGTH = b'This response do not have any content-length header'
 
 
 class GetDataHtmlSmall(LeafResource):
@@ -80,9 +79,9 @@ class PostDataJsonMixin:
             'extra-data': extra_data
         }
         for k, v in request.requestHeaders.getAllRawHeaders():
-            response['request-headers'][k.decode('utf-8')] = v[0].decode('utf-8')
+            response['request-headers'][to_unicode(k)] = to_unicode(v[0])
 
-        response_bytes = bytes(json.dumps(response), 'utf-8')
+        response_bytes = to_bytes(json.dumps(response))
         request.setHeader('Content-Type', 'application/json')
         return response_bytes
 
@@ -97,6 +96,31 @@ class PostDataJsonLarge(LeafResource, PostDataJsonMixin):
         return self.make_response(request, Data.EXTRA_LARGE)
 
 
+class Dataloss(LeafResource):
+
+    def render_GET(self, request: TxRequest):
+        request.setHeader(b"Content-Length", b"1024")
+        self.deferRequest(request, 0, self._delayed_render, request)
+        return NOT_DONE_YET
+
+    @staticmethod
+    def _delayed_render(request: TxRequest):
+        request.write(Data.DATALOSS)
+        request.finish()
+
+
+class NoContentLengthHeader(LeafResource):
+    def render_GET(self, request: TxRequest):
+        request.requestHeaders.removeHeader('Content-Length')
+        self.deferRequest(request, 0, self._delayed_render, request)
+        return NOT_DONE_YET
+
+    @staticmethod
+    def _delayed_render(request: TxRequest):
+        request.write(Data.NO_CONTENT_LENGTH)
+        request.finish()
+
+
 def get_client_certificate(key_file, certificate_file):
     with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate:
         pem = ''.join(key.readlines()) + ''.join(certificate.readlines())
@@ -118,6 +142,9 @@ class Https2ClientProtocolTestCase(TestCase):
 
         r.putChild(b'post-data-json-small', PostDataJsonSmall())
         r.putChild(b'post-data-json-large', PostDataJsonLarge())
+
+        r.putChild(b'dataloss', Dataloss())
+        r.putChild(b'no-content-length-header', NoContentLengthHeader())
         return r
 
     @inlineCallbacks
@@ -172,10 +199,10 @@ class Https2ClientProtocolTestCase(TestCase):
         return DeferredList(d_list, fireOnOneErrback=True)
 
     def _check_GET(
-            self,
-            request: Request,
-            expected_body,
-            expected_status
+        self,
+        request: Request,
+        expected_body,
+        expected_status
     ):
         def check_response(response: Response):
             self.assertEqual(response.status, expected_status)
@@ -220,11 +247,11 @@ class Https2ClientProtocolTestCase(TestCase):
         )
 
     def _check_POST_json(
-            self,
-            request: Request,
-            expected_request_body,
-            expected_extra_data,
-            expected_status: int
+        self,
+        request: Request,
+        expected_request_body,
+        expected_extra_data,
+        expected_status: int
     ):
         d = self.client.request(request)
 
@@ -237,7 +264,7 @@ class Https2ClientProtocolTestCase(TestCase):
             self.assertEqual(len(response.body), content_length)
 
             # Parse the body
-            body = json.loads(response.body.decode('utf-8'))
+            body = json.loads(to_unicode(response.body))
             self.assertIn('request-body', body)
             self.assertIn('extra-data', body)
             self.assertIn('request-headers', body)
@@ -251,11 +278,12 @@ class Https2ClientProtocolTestCase(TestCase):
             # Check if headers were sent successfully
             request_headers = body['request-headers']
             for k, v in request.headers.items():
-                k_str = k.decode('utf-8')
+                k_str = to_unicode(k)
                 self.assertIn(k_str, request_headers)
-                self.assertEqual(request_headers[k_str], v[0].decode('utf-8'))
+                self.assertEqual(request_headers[k_str], to_unicode(v[0]))
 
         d.addCallback(assert_response)
+        d.addErrback(self.fail)
         return d
 
     def test_POST_small_json(self):
@@ -310,6 +338,95 @@ class Https2ClientProtocolTestCase(TestCase):
 
         d = self.client.request(request)
         d.addCallback(assert_response)
+        d.addErrback(self.fail)
         d.cancel()
 
         return d
+
+    def test_download_maxsize_exceeded(self):
+        request = Request(url=self.get_url('/get-data-html-large'), meta={'download_maxsize': 1000})
+
+        def assert_cancelled_error(failure):
+            self.assertIsInstance(failure.value, CancelledError)
+
+        d = self.client.request(request)
+        d.addCallback(self.fail)
+        d.addErrback(assert_cancelled_error)
+        return d
+
+    # TODO: Test in multiple requests if one request fails due to dataloss
+    #  remaining request do not fail (change expected behaviour)
+    #  Can be done only when hyper-h2 don't terminate connection over
+    #  InvalidBodyLengthError check
+    def test_received_dataloss_response(self):
+        """In case when value of Header Content-Length != len(Received Data)
+        ProtocolError is raised"""
+        request = Request(url=self.get_url('/dataloss'))
+
+        def assert_failure(failure: Failure):
+            self.assertTrue(len(failure.value.reasons) > 0)
+            self.assertTrue(any(
+                isinstance(error, InvalidBodyLengthError)
+                for error in failure.value.reasons
+            ))
+
+        d = self.client.request(request)
+        d.addCallback(self.fail)
+        d.addErrback(assert_failure)
+        return d
+
+    def test_missing_content_length_header(self):
+        request = Request(url=self.get_url('/no-content-length-header'))
+
+        def assert_content_length(response: Response):
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.body, Data.NO_CONTENT_LENGTH)
+            self.assertEqual(response.request, request)
+            self.assertEqual(response.url, request.url)
+            self.assertIn('partial', response.flags)
+            self.assertNotIn('Content-Length', response.headers)
+
+        d = self.client.request(request)
+        d.addCallback(assert_content_length)
+        d.addErrback(self.fail)
+        return d
+
+    @inlineCallbacks
+    def _check_log_warnsize(
+        self,
+        request,
+        warn_pattern,
+        expected_body
+    ):
+        with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm:
+            response = yield self.client.request(request)
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.request, request)
+            self.assertEqual(response.url, request.url)
+            self.assertEqual(response.body, expected_body)
+
+            # Check the warning is raised only once for this request
+            self.assertEqual(sum(
+                len(re.findall(warn_pattern, log))
+                for log in cm.output
+            ), 1)
+
+    @inlineCallbacks
+    def test_log_expected_warnsize(self):
+        request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000})
+        warn_pattern = re.compile(
+            r'Expected response size \(\d*\) larger than '
+            r'download warn size \(1000\) in request {}'.format(request)
+        )
+
+        yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE)
+
+    @inlineCallbacks
+    def test_log_received_warnsize(self):
+        request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10})
+        warn_pattern = re.compile(
+            r'Received more \(\d*\) bytes than download '
+            r'warn size \(10\) in request {}'.format(request)
+        )
+
+        yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH)

From 26ab3e4137ddee3c643ae63c0709529efc698433 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Tue, 30 Jun 2020 06:44:20 +0530
Subject: [PATCH 061/716] feat: FIFO policy to handle large no. of requests

- add required test -- test by sending 1000 requests
- increase test timeout to 180 seconds to account for tests taking long
time
---
 scrapy/core/http2/protocol.py       | 87 +++++++++++++++++++----------
 scrapy/core/http2/stream.py         | 26 ++++++++-
 tests/test_http2_client_protocol.py | 51 ++++++++++++++++-
 3 files changed, 128 insertions(+), 36 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 0b3e5d304..4de80c05e 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -38,13 +38,16 @@ class H2ClientProtocol(Protocol):
         # Streams are stored in a dictionary keyed off their stream IDs
         self.streams: Dict[int, Stream] = {}
 
-        # Boolean to keep track the connection is made
-        # If requests are received before connection is made
-        # we keep all requests in a pool and send them as the connection
-        # is made
-        self.is_connection_made = False
+        # If requests are received before connection is made we keep
+        # all requests in a pool and send them as the connection is made
         self._pending_request_stream_pool = deque()
 
+        # Counter to keep track of opened stream. This counter
+        # is used to make sure that not more than MAX_CONCURRENT_STREAMS
+        # streams are opened which leads to ProtocolError
+        # We use simple FIFO policy to handle pending requests
+        self._active_streams = 0
+
         # Save an instance of ProtocolError raised by hyper-h2
         # We pass this instance to the streams ResponseFailed() failure
         self._protocol_error: Union[None, ProtocolError] = None
@@ -54,10 +57,46 @@ class H2ClientProtocol(Protocol):
             'ip_address': None
         }
 
+    @property
+    def is_connected(self):
+        """Boolean to keep track of the connection status.
+        This is used while initiating pending streams to make sure
+        that we initiate stream only during active HTTP/2 Connection
+        """
+        return bool(self.transport.connected)
+
+    @property
+    def allowed_max_concurrent_streams(self) -> int:
+        """We keep total two streams for client (sending data) and
+        server side (receiving data) for a single request. To be safe
+        we choose the minimum. Since this value can change in event
+        RemoteSettingsChanged we make variable a property.
+        """
+        return min(
+            self.conn.local_settings.max_concurrent_streams,
+            self.conn.remote_settings.max_concurrent_streams
+        )
+
+    def _send_pending_requests(self):
+        """Initiate all pending requests from the deque following FIFO
+        We make sure that at any time {allowed_max_concurrent_streams}
+        streams are active.
+        """
+        while (
+            self._pending_request_stream_pool
+            and self._active_streams < self.allowed_max_concurrent_streams
+            and self.is_connected
+        ):
+            self._active_streams += 1
+            stream = self._pending_request_stream_pool.popleft()
+            stream.initiate_request()
+
     def _stream_close_cb(self, stream_id: int):
         """Called when stream is closed completely
         """
-        self.streams.pop(stream_id, None)
+        self.streams.pop(stream_id)
+        self._active_streams -= 1
+        self._send_pending_requests()
 
     def _new_stream(self, request: Request):
         """Instantiates a new Stream object
@@ -75,13 +114,6 @@ class H2ClientProtocol(Protocol):
         self.streams[stream.stream_id] = stream
         return stream
 
-    def _send_pending_requests(self):
-        # TODO: handle MAX_CONCURRENT_STREAMS
-        # Initiate all pending requests
-        while self._pending_request_stream_pool:
-            stream = self._pending_request_stream_pool.popleft()
-            stream.initiate_request()
-
     def _write_to_transport(self):
         """ Write data to the underlying transport connection
         from the HTTP2 connection instance if any
@@ -89,19 +121,12 @@ class H2ClientProtocol(Protocol):
         data = self.conn.data_to_send()
         self.transport.write(data)
 
-        logger.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address']))
-
     def request(self, request: Request):
         stream = self._new_stream(request)
         d = stream.get_response()
 
-        # If connection is not yet established then add the
-        # stream to pool or initiate request
-        if self.is_connection_made:
-            stream.initiate_request()
-        else:
-            self._pending_request_stream_pool.append(stream)
-
+        # Add the stream to the request pool
+        self._pending_request_stream_pool.append(stream)
         return d
 
     def connectionMade(self):
@@ -116,7 +141,6 @@ class H2ClientProtocol(Protocol):
 
         self.conn.initiate_connection()
         self._write_to_transport()
-        self.is_connection_made = True
 
     def dataReceived(self, data):
         try:
@@ -144,7 +168,10 @@ class H2ClientProtocol(Protocol):
         # which raises `RuntimeError: dictionary changed size during iteration`
         # Hence, we copy the streams into a list.
         for stream in list(self.streams.values()):
-            stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error)
+            if stream.request_sent:
+                stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error)
+            else:
+                stream.close(StreamCloseReason.INACTIVE)
 
         self.conn.close_connection()
 
@@ -160,7 +187,6 @@ class H2ClientProtocol(Protocol):
                 triggered by sending data
         """
         for event in events:
-            logger.debug(event)
             if isinstance(event, DataReceived):
                 self.data_received(event)
             elif isinstance(event, ResponseReceived):
@@ -174,7 +200,7 @@ class H2ClientProtocol(Protocol):
             elif isinstance(event, SettingsAcknowledged):
                 self.settings_acknowledged(event)
             else:
-                logger.info("Received unhandled event {}".format(event))
+                logger.debug("Received unhandled event {}".format(event))
 
     # Event handler functions starts here
     def data_received(self, event: DataReceived):
@@ -184,8 +210,8 @@ class H2ClientProtocol(Protocol):
         self.streams[event.stream_id].receive_headers(event.headers)
 
     def settings_acknowledged(self, event: SettingsAcknowledged):
-        # Send off all the pending requests
-        # as now we have established a proper HTTP/2 connection
+        # Send off all the pending requests as now we have
+        # established a proper HTTP/2 connection
         self._send_pending_requests()
 
     def stream_ended(self, event: StreamEnded):
@@ -195,9 +221,8 @@ class H2ClientProtocol(Protocol):
         self.streams[event.stream_id].close(StreamCloseReason.RESET)
 
     def window_updated(self, event: WindowUpdated):
-        stream_id = event.stream_id
-        if stream_id != 0:
-            self.streams[stream_id].receive_window_update()
+        if event.stream_id != 0:
+            self.streams[event.stream_id].receive_window_update()
         else:
             # Send leftover data for all the streams
             for stream in self.streams.values():
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index da0181d52..19b1825e4 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -8,6 +8,7 @@ from h2.connection import H2Connection
 from h2.errors import ErrorCodes
 from h2.exceptions import StreamClosedError
 from twisted.internet.defer import Deferred, CancelledError
+from twisted.internet.error import ConnectionClosed
 from twisted.python.failure import Failure
 from twisted.web.client import ResponseFailed
 
@@ -19,6 +20,15 @@ from scrapy.responsetypes import responsetypes
 logger = logging.getLogger(__name__)
 
 
+class InactiveStreamClosed(ConnectionClosed):
+    """Connection was closed without sending request headers
+    of the stream. This happens when a stream is waiting for other
+    streams to close and connection is lost."""
+
+    def __init__(self, request: Request):
+        self.request = request
+
+
 class StreamCloseReason(Enum):
     # Received a StreamEnded event
     ENDED = 1
@@ -32,10 +42,13 @@ class StreamCloseReason(Enum):
     # Expected response body size is more than allowed limit
     MAXSIZE_EXCEEDED = 4
 
-    # When the response deferred is cancelled by the client
+    # Response deferred is cancelled by the client
     # (happens when client called response_deferred.cancel())
     CANCELLED = 5
 
+    # Connection lost and the stream was not initiated
+    INACTIVE = 6
+
 
 class Stream:
     """Represents a single HTTP/2 Stream.
@@ -108,8 +121,12 @@ class Stream:
         }
 
         def _cancel(_):
-            # Close this stream as gracefully as possible :)
-            self.reset_stream(StreamCloseReason.CANCELLED)
+            # Close this stream as gracefully as possible
+            # Check if the stream has started
+            if self.request_sent:
+                self.reset_stream(StreamCloseReason.CANCELLED)
+            else:
+                self.close(StreamCloseReason.CANCELLED)
 
         self._deferred_response = Deferred(_cancel)
 
@@ -355,6 +372,9 @@ class Stream:
                 error if error else Failure()
             ]))
 
+        elif reason is StreamCloseReason.INACTIVE:
+            self._deferred_response.errback(InactiveStreamClosed(self._request))
+
     def _fire_response_deferred(self, flags: List[str] = None):
         """Builds response from the self._response dict
         and fires the response deferred callback with the
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 0a2719d23..0cb32dda6 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -18,6 +18,7 @@ from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.static import File
 
 from scrapy.core.http2.protocol import H2ClientProtocol
+from scrapy.core.http2.stream import InactiveStreamClosed
 from scrapy.http import Request, Response, JsonRequest
 from scrapy.utils.python import to_bytes, to_unicode
 from tests.mockserver import ssl_context_factory, LeafResource
@@ -174,10 +175,14 @@ class Https2ClientProtocolTestCase(TestCase):
         client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
         self.client = yield client_endpoint.connect(h2_client_factory)
 
+        # Increase the total time taken for each tests
+        self.timeout = 180 # default is 120 seconds
+
     @inlineCallbacks
     def tearDown(self):
-        yield self.client.transport.loseConnection()
-        yield self.client.transport.abortConnection()
+        if self.client.is_connected:
+            yield self.client.transport.loseConnection()
+            yield self.client.transport.abortConnection()
         yield self.server.stopListening()
         shutil.rmtree(self.temp_directory)
 
@@ -430,3 +435,45 @@ class Https2ClientProtocolTestCase(TestCase):
         )
 
         yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH)
+
+    def test_max_concurrent_streams(self):
+        """Send 1000 requests to check if we can handle
+        very large number of request
+        """
+
+        def get_deferred():
+            return self._check_GET(
+                Request(self.get_url('/get-data-html-small')),
+                Data.HTML_SMALL,
+                200
+            )
+
+        return self._check_repeat(get_deferred, 1000)
+
+    def test_inactive_stream(self):
+        """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS
+        by default is 100. After sending the first 100 requests we close the
+        connection."""
+        d_list = []
+
+        def assert_inactive_stream(failure):
+            self.assertIsNotNone(failure.check(InactiveStreamClosed))
+
+        # Send 100 request (we do not check the result)
+        for _ in range(100):
+            d = self.client.request(Request(self.get_url('/get-data-html-small')))
+            d.addBoth(lambda _: None)
+            d_list.append(d)
+
+        # Now send 10 extra request and save the response deferred in a list
+        for _ in range(10):
+            d = self.client.request(Request(self.get_url('/get-data-html-small')))
+            d.addCallback(self.fail)
+            d.addErrback(assert_inactive_stream)
+            d_list.append(d)
+
+        # Close the connection now to fire all the extra 10 requests errback
+        # with InactiveStreamClosed
+        self.client.transport.abortConnection()
+
+        return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True)

From 50dd9271b4566785430106cfa9384d51103f73d9 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Tue, 30 Jun 2020 07:17:48 +0530
Subject: [PATCH 062/716] fix: disable redundant logs

- while testing the job exceeded the maximum log length
and was terminated
- reduce the number of requests from 20 to 10
---
 scrapy/core/http2/stream.py         |  8 --------
 tests/test_http2_client_protocol.py | 27 ++++++++++++---------------
 2 files changed, 12 insertions(+), 23 deletions(-)

diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 19b1825e4..8d0c6d94d 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -226,14 +226,6 @@ class Stream:
             self.remaining_content_length = self.remaining_content_length - chunk_size
 
         self.remaining_content_length = max(0, self.remaining_content_length)
-        logger.debug(
-            "{stream} sending {received}/{expected} data bytes ({frames} frames) to {ip_address}".format(
-                stream=self,
-                received=self.content_length - self.remaining_content_length,
-                expected=self.content_length,
-                frames=data_frames_sent,
-                ip_address=self._conn_metadata['ip_address'])
-        )
 
         # End the stream if no more data needs to be send
         if self.remaining_content_length == 0:
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 0cb32dda6..79c129d11 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -175,9 +175,6 @@ class Https2ClientProtocolTestCase(TestCase):
         client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
         self.client = yield client_endpoint.connect(h2_client_factory)
 
-        # Increase the total time taken for each tests
-        self.timeout = 180 # default is 120 seconds
-
     @inlineCallbacks
     def tearDown(self):
         if self.client.is_connected:
@@ -231,21 +228,21 @@ class Https2ClientProtocolTestCase(TestCase):
         request = Request(self.get_url('/get-data-html-large'))
         return self._check_GET(request, Data.HTML_LARGE, 200)
 
-    def _check_GET_x20(self, *args, **kwargs):
+    def _check_GET_x10(self, *args, **kwargs):
         def get_deferred():
             return self._check_GET(*args, **kwargs)
 
-        return self._check_repeat(get_deferred, 20)
+        return self._check_repeat(get_deferred, 10)
 
-    def test_GET_small_body_x20(self):
-        return self._check_GET_x20(
+    def test_GET_small_body_x10(self):
+        return self._check_GET_x10(
             Request(self.get_url('/get-data-html-small')),
             Data.HTML_SMALL,
             200
         )
 
-    def test_GET_large_body_x20(self):
-        return self._check_GET_x20(
+    def test_GET_large_body_x10(self):
+        return self._check_GET_x10(
             Request(self.get_url('/get-data-html-large')),
             Data.HTML_LARGE,
             200
@@ -309,24 +306,24 @@ class Https2ClientProtocolTestCase(TestCase):
             200
         )
 
-    def _check_POST_json_x20(self, *args, **kwargs):
+    def _check_POST_json_x10(self, *args, **kwargs):
         def get_deferred():
             return self._check_POST_json(*args, **kwargs)
 
-        return self._check_repeat(get_deferred, 20)
+        return self._check_repeat(get_deferred, 10)
 
-    def test_POST_small_json_x20(self):
+    def test_POST_small_json_x10(self):
         request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL)
-        return self._check_POST_json_x20(
+        return self._check_POST_json_x10(
             request,
             Data.JSON_SMALL,
             Data.EXTRA_SMALL,
             200
         )
 
-    def test_POST_large_json_x20(self):
+    def test_POST_large_json_x10(self):
         request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE)
-        return self._check_POST_json_x20(
+        return self._check_POST_json_x10(
             request,
             Data.JSON_LARGE,
             Data.EXTRA_LARGE,

From 7b1ad995a4996babf9a019815dd7256a1cbfa044 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 1 Jul 2020 10:45:36 +0530
Subject: [PATCH 063/716] test: query params, certificate & ip_address

- refactor from str.format() to f-strings
---
 scrapy/core/http2/protocol.py       | 12 ++--
 scrapy/core/http2/stream.py         | 46 +++++---------
 scrapy/core/http2/types.py          |  6 +-
 setup.py                            |  7 ++-
 tests/test_http2_client_protocol.py | 96 +++++++++++++++++++++++------
 5 files changed, 108 insertions(+), 59 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 4de80c05e..3438c99f0 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -122,6 +122,9 @@ class H2ClientProtocol(Protocol):
         self.transport.write(data)
 
     def request(self, request: Request):
+        if not isinstance(request, Request):
+            raise TypeError(f'Expected type scrapy.http.Request but received {request.__class__.__name__}')
+
         stream = self._new_stream(request)
         d = stream.get_response()
 
@@ -134,9 +137,7 @@ class H2ClientProtocol(Protocol):
         sending some data now: we should open with the connection preamble.
         """
         self.destination = self.transport.getPeer()
-        logger.info('Connection made to {}'.format(self.destination))
-
-        self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
+        logger.info(f'Connection made to {self.destination}')
         self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host)
 
         self.conn.initiate_connection()
@@ -200,7 +201,7 @@ class H2ClientProtocol(Protocol):
             elif isinstance(event, SettingsAcknowledged):
                 self.settings_acknowledged(event)
             else:
-                logger.debug("Received unhandled event {}".format(event))
+                logger.debug(f'Received unhandled event {event}')
 
     # Event handler functions starts here
     def data_received(self, event: DataReceived):
@@ -214,6 +215,9 @@ class H2ClientProtocol(Protocol):
         # established a proper HTTP/2 connection
         self._send_pending_requests()
 
+        # Update certificate when our HTTP/2 connection is established
+        self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
+
     def stream_ended(self, event: StreamEnded):
         self.streams[event.stream_id].close(StreamCloseReason.ENDED)
 
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 8d0c6d94d..f4a90a753 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -131,7 +131,7 @@ class Stream:
         self._deferred_response = Deferred(_cancel)
 
     def __str__(self):
-        return "Stream(id={})".format(repr(self.stream_id))
+        return f'Stream(id={self.stream_id})'
 
     __repr__ = __str__
 
@@ -167,13 +167,15 @@ class Stream:
         url = urlparse(self._request.url)
 
         # Make sure pseudo-headers comes before all the other headers
+        path = url.path
+        if url.query:
+            path += '?' + url.query
+
         headers = [
             (':method', self._request.method),
             (':authority', url.netloc),
-
-            # TODO: Check if scheme can be 'http' for HTTP/2 ?
             (':scheme', 'https'),
-            (':path', url.path),
+            (':path', path),
         ]
 
         for name, value in self._request.headers.items():
@@ -253,14 +255,9 @@ class Stream:
 
         if self._log_warnsize:
             self._reached_warnsize = True
-            warning_msg = 'Received more ({bytes}) bytes than download ' \
-                          + 'warn size ({warnsize}) in request {request}'
-            warning_args = {
-                'bytes': self._response['flow_controlled_size'],
-                'warnsize': self._download_warnsize,
-                'request': self._request
-            }
-            logger.warning(warning_msg.format(**warning_args))
+            warning_msg = f"Received more ({self._response['flow_controlled_size']}) bytes than download " \
+                          + f'warn size ({self._download_warnsize}) in request {self._request}'
+            logger.warning(warning_msg)
 
         # Acknowledge the data received
         self._conn.acknowledge_received_data(
@@ -280,14 +277,9 @@ class Stream:
 
         if self._log_warnsize:
             self._reached_warnsize = True
-            warning_msg = 'Expected response size ({size}) larger than ' \
-                          + 'download warn size ({warnsize}) in request {request}'
-            warning_args = {
-                'size': expected_size,
-                'warnsize': self._download_warnsize,
-                'request': self._request
-            }
-            logger.warning(warning_msg.format(**warning_args))
+            warning_msg = f'Expected response size ({expected_size}) larger than ' \
+                          + f'download warn size ({self._download_warnsize}) in request {self._request}'
+            logger.warning(warning_msg)
 
     def reset_stream(self, reason=StreamCloseReason.RESET):
         """Close this stream by sending a RST_FRAME to the remote peer"""
@@ -332,16 +324,10 @@ class Stream:
         # having Content-Length)
         if reason is StreamCloseReason.MAXSIZE_EXCEEDED:
             expected_size = int(self._response['headers'].get(b'Content-Length', -1))
-            error_msg = ("Cancelling download of {url}: expected response "
-                         "size ({size}) larger than download max size ({maxsize}).")
-            error_args = {
-                'url': self._request.url,
-                'size': expected_size,
-                'maxsize': self._download_maxsize
-            }
-
-            logger.error(error_msg, error_args)
-            self._deferred_response.errback(CancelledError(error_msg.format(**error_args)))
+            error_msg = f'Cancelling download of {self._request.url}: expected response ' \
+                        f'size ({expected_size}) larger than download max size ({self._download_maxsize}).'
+            logger.error(error_msg)
+            self._deferred_response.errback(CancelledError(error_msg))
 
         elif reason is StreamCloseReason.ENDED:
             self._fire_response_deferred(flags)
diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py
index f28bf9472..c0961cd3a 100644
--- a/scrapy/core/http2/types.py
+++ b/scrapy/core/http2/types.py
@@ -1,6 +1,6 @@
 from io import BytesIO
 from ipaddress import IPv4Address, IPv6Address
-from typing import Union
+from typing import Union, Optional
 
 from twisted.internet.ssl import Certificate
 # for python < 3.8 -- typing.TypedDict is undefined
@@ -13,8 +13,8 @@ class H2ConnectionMetadataDict(TypedDict):
     """Some meta data of this connection
     initialized when connection is successfully made
     """
-    certificate: Union[None, Certificate]
-    ip_address: Union[None, IPv4Address, IPv6Address]
+    certificate: Optional[Certificate]
+    ip_address: Optional[Union[IPv4Address, IPv6Address]]
 
 
 class H2ResponseDict(TypedDict):
diff --git a/setup.py b/setup.py
index 8e50733e6..47c5906e4 100644
--- a/setup.py
+++ b/setup.py
@@ -1,8 +1,8 @@
 from os.path import dirname, join
-
 from pkg_resources import parse_version
 from setuptools import setup, find_packages, __version__ as setuptools_version
 
+
 with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f:
     version = f.read().decode('ascii').strip()
 
@@ -25,11 +25,12 @@ if has_environment_marker_platform_impl_support():
         'PyPyDispatcher>=2.1.0',
     ]
 
+
 setup(
     name='Scrapy',
     version=version,
     url='https://scrapy.org',
-    project_urls={
+    project_urls = {
         'Documentation': 'https://docs.scrapy.org/',
         'Source': 'https://github.com/scrapy/scrapy',
         'Tracker': 'https://github.com/scrapy/scrapy/issues',
@@ -83,4 +84,4 @@ setup(
         'typing_extensions>=3.7'
     ],
     extras_require=extras_require,
-)
+)
\ No newline at end of file
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 79c129d11..c6a9bd5fb 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -4,13 +4,15 @@ import random
 import re
 import shutil
 import string
+from ipaddress import IPv4Address
+from urllib.parse import urlencode
 
 from h2.exceptions import InvalidBodyLengthError
 from twisted.internet import reactor
 from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint
 from twisted.internet.protocol import Factory
-from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate
+from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate
 from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
 from twisted.web.http import Request as TxRequest
@@ -21,7 +23,7 @@ from scrapy.core.http2.protocol import H2ClientProtocol
 from scrapy.core.http2.stream import InactiveStreamClosed
 from scrapy.http import Request, Response, JsonRequest
 from scrapy.utils.python import to_bytes, to_unicode
-from tests.mockserver import ssl_context_factory, LeafResource
+from tests.mockserver import ssl_context_factory, LeafResource, Status
 
 
 def generate_random_string(size):
@@ -32,10 +34,10 @@ def generate_random_string(size):
 
 
 def make_html_body(val):
-    response = '''<html>
+    response = f'''<html>
 <h1>Hello from HTTP2<h1>
-<p>{}</p>
-</html>'''.format(val)
+<p>{val}</p>
+</html>'''
     return to_bytes(response)
 
 
@@ -122,6 +124,17 @@ class NoContentLengthHeader(LeafResource):
         request.finish()
 
 
+class QueryParams(LeafResource):
+    def render_GET(self, request: TxRequest):
+        request.setHeader('Content-Type', 'application/json')
+
+        query_params = {}
+        for k, v in request.args.items():
+            query_params[to_unicode(k)] = to_unicode(v[0])
+
+        return to_bytes(json.dumps(query_params))
+
+
 def get_client_certificate(key_file, certificate_file):
     with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate:
         pem = ''.join(key.readlines()) + ''.join(certificate.readlines())
@@ -146,6 +159,8 @@ class Https2ClientProtocolTestCase(TestCase):
 
         r.putChild(b'dataloss', Dataloss())
         r.putChild(b'no-content-length-header', NoContentLengthHeader())
+        r.putChild(b'status', Status())
+        r.putChild(b'query-params', QueryParams())
         return r
 
     @inlineCallbacks
@@ -189,7 +204,7 @@ class Https2ClientProtocolTestCase(TestCase):
         :return: Complete url
         """
         assert len(path) > 0 and (path[0] == '/' or path[0] == '&')
-        return "{}://{}:{}{}".format(self.scheme, self.hostname, self.port_number, path)
+        return f'{self.scheme}://{self.hostname}:{self.port_number}{path}'
 
     @staticmethod
     def _check_repeat(get_deferred, count):
@@ -210,7 +225,6 @@ class Https2ClientProtocolTestCase(TestCase):
             self.assertEqual(response.status, expected_status)
             self.assertEqual(response.body, expected_body)
             self.assertEqual(response.request, request)
-            self.assertEqual(response.url, request.url)
 
             content_length = int(response.headers.get('Content-Length'))
             self.assertEqual(len(response.body), content_length)
@@ -260,7 +274,6 @@ class Https2ClientProtocolTestCase(TestCase):
         def assert_response(response: Response):
             self.assertEqual(response.status, expected_status)
             self.assertEqual(response.request, request)
-            self.assertEqual(response.url, request.url)
 
             content_length = int(response.headers.get('Content-Length'))
             self.assertEqual(len(response.body), content_length)
@@ -336,7 +349,6 @@ class Https2ClientProtocolTestCase(TestCase):
         def assert_response(response: Response):
             self.assertEqual(response.status, 499)
             self.assertEqual(response.request, request)
-            self.assertEqual(response.url, request.url)
 
         d = self.client.request(request)
         d.addCallback(assert_response)
@@ -356,10 +368,6 @@ class Https2ClientProtocolTestCase(TestCase):
         d.addErrback(assert_cancelled_error)
         return d
 
-    # TODO: Test in multiple requests if one request fails due to dataloss
-    #  remaining request do not fail (change expected behaviour)
-    #  Can be done only when hyper-h2 don't terminate connection over
-    #  InvalidBodyLengthError check
     def test_received_dataloss_response(self):
         """In case when value of Header Content-Length != len(Received Data)
         ProtocolError is raised"""
@@ -384,7 +392,6 @@ class Https2ClientProtocolTestCase(TestCase):
             self.assertEqual(response.status, 200)
             self.assertEqual(response.body, Data.NO_CONTENT_LENGTH)
             self.assertEqual(response.request, request)
-            self.assertEqual(response.url, request.url)
             self.assertIn('partial', response.flags)
             self.assertNotIn('Content-Length', response.headers)
 
@@ -404,7 +411,6 @@ class Https2ClientProtocolTestCase(TestCase):
             response = yield self.client.request(request)
             self.assertEqual(response.status, 200)
             self.assertEqual(response.request, request)
-            self.assertEqual(response.url, request.url)
             self.assertEqual(response.body, expected_body)
 
             # Check the warning is raised only once for this request
@@ -417,8 +423,8 @@ class Https2ClientProtocolTestCase(TestCase):
     def test_log_expected_warnsize(self):
         request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000})
         warn_pattern = re.compile(
-            r'Expected response size \(\d*\) larger than '
-            r'download warn size \(1000\) in request {}'.format(request)
+            rf'Expected response size \(\d*\) larger than '
+            rf'download warn size \(1000\) in request {request}'
         )
 
         yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE)
@@ -427,8 +433,8 @@ class Https2ClientProtocolTestCase(TestCase):
     def test_log_received_warnsize(self):
         request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10})
         warn_pattern = re.compile(
-            r'Received more \(\d*\) bytes than download '
-            r'warn size \(10\) in request {}'.format(request)
+            rf'Received more \(\d*\) bytes than download '
+            rf'warn size \(10\) in request {request}'
         )
 
         yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH)
@@ -474,3 +480,55 @@ class Https2ClientProtocolTestCase(TestCase):
         self.client.transport.abortConnection()
 
         return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True)
+
+    def test_invalid_request_type(self):
+        with self.assertRaises(TypeError):
+            self.client.request('https://InvalidDataTypePassed.com')
+
+    def test_query_parameters(self):
+        params = {
+            'a': generate_random_string(20),
+            'b': generate_random_string(20),
+            'c': generate_random_string(20),
+            'd': generate_random_string(20)
+        }
+        request = Request(self.get_url(f'/query-params?{urlencode(params)}'))
+
+        def assert_query_params(response: Response):
+            data = json.loads(to_unicode(response.body))
+            self.assertEqual(data, params)
+
+        d = self.client.request(request)
+        d.addCallback(assert_query_params)
+        d.addErrback(self.fail)
+
+        return d
+
+    def test_status_codes(self):
+        def assert_response_status(response: Response, expected_status: int):
+            self.assertEqual(response.status, expected_status)
+
+        d_list = []
+        for status in [200, 404]:
+            request = Request(self.get_url(f'/status?n={status}'))
+            d = self.client.request(request)
+            d.addCallback(assert_response_status, status)
+            d.addErrback(self.fail)
+            d_list.append(d)
+
+        return DeferredList(d_list, fireOnOneErrback=True)
+
+    def test_response_has_correct_certificate_ip_address(self):
+        request = Request(self.get_url('/status?n=200'))
+
+        def assert_metadata(response: Response):
+            self.assertEqual(response.request, request)
+            self.assertIsInstance(response.certificate, Certificate)
+            self.assertIsInstance(response.ip_address, IPv4Address)
+            self.assertEqual(str(response.ip_address), '127.0.0.1')
+
+        d = self.client.request(request)
+        d.addCallback(assert_metadata)
+        d.addErrback(self.fail)
+
+        return d

From 7fc80671a84144ad36c3d5332aa822ef3be00781 Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Wed, 1 Jul 2020 13:32:17 +0530
Subject: [PATCH 064/716] Update schemaless URI support

---
 scrapy/core/downloader/handlers/http11.py | 12 ++++++++----
 scrapy/core/downloader/webclient.py       |  3 +++
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 73e56c87d..15de8cdbd 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -7,7 +7,7 @@ import warnings
 from contextlib import suppress
 from io import BytesIO
 from time import time
-from urllib.parse import urldefrag, urlparse
+from urllib.parse import urldefrag, urlunparse
 
 from twisted.internet import defer, protocol, ssl
 from twisted.internet.endpoints import TCP4ClientEndpoint
@@ -255,7 +255,7 @@ class ScrapyProxyAgent(Agent):
             bindAddress=bindAddress,
             pool=pool,
         )
-        self._proxyURI = URI.fromBytes(urlparse(proxyURI)._replace(scheme=b'http').geturl())
+        self._proxyURI = URI.fromBytes(proxyURI)
 
     def request(self, method, uri, headers=None, bodyProducer=None):
         """
@@ -297,7 +297,7 @@ class ScrapyAgent:
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
         proxy = request.meta.get('proxy')
         if proxy:
-            _, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
+            proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
             scheme = _parse(request.url)[0]
             proxyHost = to_unicode(proxyHost)
             omitConnectTunnel = b'noconnect' in proxyParams
@@ -319,9 +319,13 @@ class ScrapyAgent:
                     pool=self._pool,
                 )
             else:
+                proxyScheme = b'http' if not proxyScheme else proxyScheme
+                proxyHost = to_bytes(proxyHost, encoding='ascii')
+                proxyPort = to_bytes(str(proxyPort), encoding='ascii')
+                proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
                 return self._ProxyAgent(
                     reactor=reactor,
-                    proxyURI=to_bytes(proxy, encoding='ascii'),
+                    proxyURI=to_bytes(proxyURI, encoding='ascii'),
                     connectTimeout=timeout,
                     bindAddress=bindaddress,
                     pool=self._pool,
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 355045d74..af49e78ce 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,5 +1,6 @@
 from time import time
 from urllib.parse import urlparse, urlunparse, urldefrag
+from re import match
 
 from twisted.web.client import HTTPClientFactory
 from twisted.web.http import HTTPClient
@@ -32,6 +33,8 @@ def _parse(url):
     and is ascii-only.
     """
     url = url.strip()
+    if not match(r'^\w+://', url):
+        url = '//' + url
     parsed = urlparse(url)
     return _parsed_url_args(parsed)
 

From 006a945214422da22f645b4416263124377adc5e Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Wed, 1 Jul 2020 13:32:58 +0530
Subject: [PATCH 065/716] Update schemaless http proxy test

---
 tests/test_downloader_handlers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 5854659dd..9441be736 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -741,7 +741,7 @@ class Http11ProxyTestCase(HttpProxyTestCase):
             self.assertEqual(response.url, request.url)
             self.assertEqual(response.body, b'http://example.com')
 
-        http_proxy = self.getURL('').replace('http:', '')
+        http_proxy = self.getURL('').replace('http://', '')
         request = Request('http://example.com', meta={'proxy': http_proxy})
         return self.download_request(request, Spider('foo')).addCallback(_test)
 

From 065b9b1170fe249fbfce51c87631e5572127df21 Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Wed, 1 Jul 2020 15:53:29 +0530
Subject: [PATCH 066/716] Update regex import

---
 scrapy/core/downloader/webclient.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index af49e78ce..8b6f87c3f 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,6 +1,6 @@
+import re
 from time import time
 from urllib.parse import urlparse, urlunparse, urldefrag
-from re import match
 
 from twisted.web.client import HTTPClientFactory
 from twisted.web.http import HTTPClient
@@ -33,7 +33,7 @@ def _parse(url):
     and is ascii-only.
     """
     url = url.strip()
-    if not match(r'^\w+://', url):
+    if not re.match(r'^\w+://', url):
         url = '//' + url
     parsed = urlparse(url)
     return _parsed_url_args(parsed)

From c361fe0d3b80ff8a9f88adc05e730d5e469db225 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 1 Jul 2020 18:14:44 +0530
Subject: [PATCH 067/716] feat: check for invalid hostname

- Initiating requests having hostname or (ip_address, port) different
from the peer to which HTTP/2 connection is made can lead to closing the
whole connection and close out all the pending streams.
- This change aims to fix that problem
- Add required tests
- Save hostname & port in H2ConnectionMetadataDict
---
 scrapy/core/http2/protocol.py       | 26 ++++++------
 scrapy/core/http2/stream.py         | 50 +++++++++++++++++++---
 scrapy/core/http2/types.py          | 11 +++++
 tests/test_http2_client_protocol.py | 64 +++++++++++++++++++++++------
 4 files changed, 118 insertions(+), 33 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 3438c99f0..5de516482 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -2,7 +2,7 @@ import ipaddress
 import itertools
 import logging
 from collections import deque
-from typing import Union, Dict
+from typing import Dict, Optional
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
@@ -26,10 +26,6 @@ class H2ClientProtocol(Protocol):
         config = H2Configuration(client_side=True, header_encoding='utf-8')
         self.conn = H2Connection(config=config)
 
-        # Address of the server we are connected to
-        # these are updated when connection is successfully made
-        self.destination = None
-
         # ID of the next request stream
         # Following the convention made by hyper-h2 each client ID
         # will be odd.
@@ -50,11 +46,13 @@ class H2ClientProtocol(Protocol):
 
         # Save an instance of ProtocolError raised by hyper-h2
         # We pass this instance to the streams ResponseFailed() failure
-        self._protocol_error: Union[None, ProtocolError] = None
+        self._protocol_error: Optional[ProtocolError] = None
 
         self._metadata: H2ConnectionMetadataDict = {
             'certificate': None,
-            'ip_address': None
+            'ip_address': None,
+            'hostname': None,
+            'port': None
         }
 
     @property
@@ -123,7 +121,7 @@ class H2ClientProtocol(Protocol):
 
     def request(self, request: Request):
         if not isinstance(request, Request):
-            raise TypeError(f'Expected type scrapy.http.Request but received {request.__class__.__name__}')
+            raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__name__}')
 
         stream = self._new_stream(request)
         d = stream.get_response()
@@ -136,9 +134,11 @@ class H2ClientProtocol(Protocol):
         """Called by Twisted when the connection is established. We can start
         sending some data now: we should open with the connection preamble.
         """
-        self.destination = self.transport.getPeer()
-        logger.info(f'Connection made to {self.destination}')
-        self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host)
+        destination = self.transport.getPeer()
+        logger.debug('Connection made to {}'.format(destination))
+        self._metadata['ip_address'] = ipaddress.ip_address(destination.host)
+        self._metadata['port'] = destination.port
+        self._metadata['hostname'] = self.transport.transport.addr[0]
 
         self.conn.initiate_connection()
         self._write_to_transport()
@@ -148,8 +148,6 @@ class H2ClientProtocol(Protocol):
             events = self.conn.receive_data(data)
             self._handle_events(events)
         except ProtocolError as e:
-            # TODO: In case of InvalidBodyLengthError -- terminate only one stream
-
             # Save this error as ultimately the connection will be dropped
             # internally by hyper-h2. Saved error will be passed to all the streams
             # closed with the connection.
@@ -201,7 +199,7 @@ class H2ClientProtocol(Protocol):
             elif isinstance(event, SettingsAcknowledged):
                 self.settings_acknowledged(event)
             else:
-                logger.debug(f'Received unhandled event {event}')
+                logger.debug('Received unhandled event {}'.format(event))
 
     # Event handler functions starts here
     def data_received(self, event: DataReceived):
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index f4a90a753..cc751b682 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -29,6 +29,17 @@ class InactiveStreamClosed(ConnectionClosed):
         self.request = request
 
 
+class InvalidHostname(Exception):
+
+    def __init__(self, request: Request, expected_hostname, expected_netloc):
+        self.request = request
+        self.expected_hostname = expected_hostname
+        self.expected_netloc = expected_netloc
+
+    def __str__(self):
+        return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}'
+
+
 class StreamCloseReason(Enum):
     # Received a StreamEnded event
     ENDED = 1
@@ -49,6 +60,10 @@ class StreamCloseReason(Enum):
     # Connection lost and the stream was not initiated
     INACTIVE = 6
 
+    # The hostname of the request is not same as of connected peer hostname
+    # As a result sending this request will the end the connection
+    INVALID_HOSTNAME = 7
+
 
 class Stream:
     """Represents a single HTTP/2 Stream.
@@ -163,6 +178,15 @@ class Stream:
         """
         return self._deferred_response
 
+    def check_request_url(self) -> bool:
+        # Make sure that we are sending the request to the correct URL
+        url = urlparse(self._request.url)
+        return (
+            url.netloc == self._conn_metadata['hostname']
+            or url.netloc == f'{self._conn_metadata["hostname"]}:{self._conn_metadata["port"]}'
+            or url.netloc == f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}'
+        )
+
     def _get_request_headers(self):
         url = urlparse(self._request.url)
 
@@ -184,10 +208,15 @@ class Stream:
         return headers
 
     def initiate_request(self):
-        headers = self._get_request_headers()
-        self._conn.send_headers(self.stream_id, headers, end_stream=False)
-        self.request_sent = True
-        self.send_data()
+        if self.check_request_url():
+            headers = self._get_request_headers()
+            self._conn.send_headers(self.stream_id, headers, end_stream=False)
+            self.request_sent = True
+            self.send_data()
+        else:
+            # Close this stream calling the response errback
+            # Note that we have not sent any headers
+            self.close(StreamCloseReason.INVALID_HOSTNAME)
 
     def send_data(self):
         """Called immediately after the headers are sent. Here we send all the
@@ -310,6 +339,9 @@ class Stream:
         if self.stream_closed_server:
             raise StreamClosedError(self.stream_id)
 
+        if not isinstance(reason, StreamCloseReason):
+            raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__name__}')
+
         self._cb_close(self.stream_id)
         self.stream_closed_server = True
 
@@ -353,6 +385,13 @@ class Stream:
         elif reason is StreamCloseReason.INACTIVE:
             self._deferred_response.errback(InactiveStreamClosed(self._request))
 
+        elif reason is StreamCloseReason.INVALID_HOSTNAME:
+            self._deferred_response.errback(InvalidHostname(
+                self._request,
+                self._conn_metadata['hostname'],
+                f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}'
+            ))
+
     def _fire_response_deferred(self, flags: List[str] = None):
         """Builds response from the self._response dict
         and fires the response deferred callback with the
@@ -365,10 +404,9 @@ class Stream:
             body=body
         )
 
-        status = self._response['headers'][':status']
         response = response_cls(
             url=self._request.url,
-            status=status,
+            status=self._response['headers'][':status'],
             headers=self._response['headers'],
             body=body,
             request=self._request,
diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py
index c0961cd3a..dd7b1187b 100644
--- a/scrapy/core/http2/types.py
+++ b/scrapy/core/http2/types.py
@@ -14,8 +14,19 @@ class H2ConnectionMetadataDict(TypedDict):
     initialized when connection is successfully made
     """
     certificate: Optional[Certificate]
+
+    # Address of the server we are connected to which
+    # is updated when HTTP/2 connection is  made successfully
     ip_address: Optional[Union[IPv4Address, IPv6Address]]
 
+    # Name of the peer HTTP/2 connection is established
+    hostname: Optional[str]
+
+    port: Optional[int]
+
+    # Both ip_address and hostname are used by the Stream before
+    # initiating the request to verify that the base address
+
 
 class H2ResponseDict(TypedDict):
     # Data received frame by frame from the server is appended
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index c6a9bd5fb..8f9fbe6fd 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -10,7 +10,7 @@ from urllib.parse import urlencode
 from h2.exceptions import InvalidBodyLengthError
 from twisted.internet import reactor
 from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError
-from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint
+from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
 from twisted.internet.protocol import Factory
 from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate
 from twisted.python.failure import Failure
@@ -20,7 +20,7 @@ from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.static import File
 
 from scrapy.core.http2.protocol import H2ClientProtocol
-from scrapy.core.http2.stream import InactiveStreamClosed
+from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
 from scrapy.http import Request, Response, JsonRequest
 from scrapy.utils.python import to_bytes, to_unicode
 from tests.mockserver import ssl_context_factory, LeafResource, Status
@@ -135,7 +135,7 @@ class QueryParams(LeafResource):
         return to_bytes(json.dumps(query_params))
 
 
-def get_client_certificate(key_file, certificate_file):
+def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
     with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate:
         pem = ''.join(key.readlines()) + ''.join(certificate.readlines())
 
@@ -171,19 +171,16 @@ class Https2ClientProtocolTestCase(TestCase):
 
         # Start server for testing
         self.hostname = u'localhost'
-        if self.scheme == 'https':
-            context_factory = ssl_context_factory(self.key_file, self.certificate_file)
-            server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname)
-        else:
-            server_endpoint = TCP4ServerEndpoint(reactor, 0, interface=self.hostname)
+        context_factory = ssl_context_factory(self.key_file, self.certificate_file)
+        server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname)
         self.server = yield server_endpoint.listen(self.site)
         self.port_number = self.server.getHost().port
 
         # Connect H2 client with server
-        client_certificate = get_client_certificate(self.key_file, self.certificate_file)
+        self.client_certificate = get_client_certificate(self.key_file, self.certificate_file)
         client_options = optionsForClientTLS(
             hostname=self.hostname,
-            trustRoot=client_certificate,
+            trustRoot=self.client_certificate,
             acceptableProtocols=[b'h2']
         )
         h2_client_factory = Factory.forProtocol(H2ClientProtocol)
@@ -440,8 +437,8 @@ class Https2ClientProtocolTestCase(TestCase):
         yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH)
 
     def test_max_concurrent_streams(self):
-        """Send 1000 requests to check if we can handle
-        very large number of request
+        """Send 500 requests at one to check if we can handle
+        very large number of request.
         """
 
         def get_deferred():
@@ -451,7 +448,7 @@ class Https2ClientProtocolTestCase(TestCase):
                 200
             )
 
-        return self._check_repeat(get_deferred, 1000)
+        return self._check_repeat(get_deferred, 500)
 
     def test_inactive_stream(self):
         """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS
@@ -524,6 +521,10 @@ class Https2ClientProtocolTestCase(TestCase):
         def assert_metadata(response: Response):
             self.assertEqual(response.request, request)
             self.assertIsInstance(response.certificate, Certificate)
+            self.assertIsNotNone(response.certificate.original)
+            self.assertEqual(response.certificate.getIssuer(), self.client_certificate.getIssuer())
+            self.assertTrue(response.certificate.getPublicKey().matches(self.client_certificate.getPublicKey()))
+
             self.assertIsInstance(response.ip_address, IPv4Address)
             self.assertEqual(str(response.ip_address), '127.0.0.1')
 
@@ -532,3 +533,40 @@ class Https2ClientProtocolTestCase(TestCase):
         d.addErrback(self.fail)
 
         return d
+
+    def _check_invalid_netloc(self, url):
+        request = Request(url)
+
+        def assert_invalid_hostname(failure: Failure):
+            self.assertIsNotNone(failure.check(InvalidHostname))
+            error_msg = str(failure.value)
+            self.assertIn('localhost', error_msg)
+            self.assertIn('127.0.0.1', error_msg)
+            self.assertIn(str(request), error_msg)
+
+        d = self.client.request(request)
+        d.addCallback(self.fail)
+        d.addErrback(assert_invalid_hostname)
+        return d
+
+    def test_invalid_hostname(self):
+        return self._check_invalid_netloc('https://notlocalhost.notlocalhostdomain')
+
+    def test_invalid_host_port(self):
+        port = self.port_number + 1
+        return self._check_invalid_netloc(f'https://127.0.0.1:{port}')
+
+    def test_connection_stays_with_invalid_requests(self):
+        d_list = [
+            self.test_invalid_hostname(),
+            self.test_invalid_host_port(),
+            self._check_GET(Request(self.get_url('/get-data-html-small')), Data.HTML_SMALL, 200),
+            self._check_POST_json(
+                JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL),
+                Data.JSON_SMALL,
+                Data.EXTRA_SMALL,
+                200
+            )
+        ]
+
+        return DeferredList(d_list, fireOnOneErrback=True)

From 4acdc2e5d623c6330235647aaad817e77eaad800 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 1 Jul 2020 20:15:33 +0530
Subject: [PATCH 068/716] refactor: use __qualname__, () for large strings

---
 scrapy/core/http2/protocol.py       |  2 +-
 scrapy/core/http2/stream.py         | 24 +++++++++++++++---------
 tests/test_http2_client_protocol.py |  9 ++-------
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 5de516482..a3dfdb76e 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -121,7 +121,7 @@ class H2ClientProtocol(Protocol):
 
     def request(self, request: Request):
         if not isinstance(request, Request):
-            raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__name__}')
+            raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}')
 
         stream = self._new_stream(request)
         d = stream.get_response()
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index cc751b682..f45ddc04e 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -146,7 +146,7 @@ class Stream:
         self._deferred_response = Deferred(_cancel)
 
     def __str__(self):
-        return f'Stream(id={self.stream_id})'
+        return f'Stream(id={self.stream_id!r})'
 
     __repr__ = __str__
 
@@ -190,11 +190,11 @@ class Stream:
     def _get_request_headers(self):
         url = urlparse(self._request.url)
 
-        # Make sure pseudo-headers comes before all the other headers
         path = url.path
         if url.query:
             path += '?' + url.query
 
+        # Make sure pseudo-headers comes before all the other headers
         headers = [
             (':method', self._request.method),
             (':authority', url.netloc),
@@ -284,8 +284,10 @@ class Stream:
 
         if self._log_warnsize:
             self._reached_warnsize = True
-            warning_msg = f"Received more ({self._response['flow_controlled_size']}) bytes than download " \
-                          + f'warn size ({self._download_warnsize}) in request {self._request}'
+            warning_msg = (
+                f'Received more ({self._response["flow_controlled_size"]}) bytes than download '
+                f'warn size ({self._download_warnsize}) in request {self._request}'
+            )
             logger.warning(warning_msg)
 
         # Acknowledge the data received
@@ -306,8 +308,10 @@ class Stream:
 
         if self._log_warnsize:
             self._reached_warnsize = True
-            warning_msg = f'Expected response size ({expected_size}) larger than ' \
-                          + f'download warn size ({self._download_warnsize}) in request {self._request}'
+            warning_msg = (
+                f'Expected response size ({expected_size}) larger than '
+                f'download warn size ({self._download_warnsize}) in request {self._request}'
+            )
             logger.warning(warning_msg)
 
     def reset_stream(self, reason=StreamCloseReason.RESET):
@@ -340,7 +344,7 @@ class Stream:
             raise StreamClosedError(self.stream_id)
 
         if not isinstance(reason, StreamCloseReason):
-            raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__name__}')
+            raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}')
 
         self._cb_close(self.stream_id)
         self.stream_closed_server = True
@@ -356,8 +360,10 @@ class Stream:
         # having Content-Length)
         if reason is StreamCloseReason.MAXSIZE_EXCEEDED:
             expected_size = int(self._response['headers'].get(b'Content-Length', -1))
-            error_msg = f'Cancelling download of {self._request.url}: expected response ' \
-                        f'size ({expected_size}) larger than download max size ({self._download_maxsize}).'
+            error_msg = (
+                f'Cancelling download of {self._request.url}: expected response '
+                f'size ({expected_size}) larger than download max size ({self._download_maxsize}).'
+            )
             logger.error(error_msg)
             self._deferred_response.errback(CancelledError(error_msg))
 
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 8f9fbe6fd..ca8a629b1 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -560,13 +560,8 @@ class Https2ClientProtocolTestCase(TestCase):
         d_list = [
             self.test_invalid_hostname(),
             self.test_invalid_host_port(),
-            self._check_GET(Request(self.get_url('/get-data-html-small')), Data.HTML_SMALL, 200),
-            self._check_POST_json(
-                JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL),
-                Data.JSON_SMALL,
-                Data.EXTRA_SMALL,
-                200
-            )
+            self.test_GET_small_body(),
+            self.test_POST_small_json()
         ]
 
         return DeferredList(d_list, fireOnOneErrback=True)

From a94b30342a451b94e7f358f68ce1b1adc20723f9 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Mon, 6 Jul 2020 12:49:12 +0530
Subject: [PATCH 069/716] test: reduce test data size to 1MB

---
 scrapy/core/http2/stream.py         | 2 --
 tests/test_http2_client_protocol.py | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index f45ddc04e..1c856ff68 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -243,7 +243,6 @@ class Stream:
         bytes_to_send_size = min(window_size, self.remaining_content_length)
 
         # We now need to send a number of data frames.
-        data_frames_sent = 0
         while bytes_to_send_size > 0:
             chunk_size = min(bytes_to_send_size, max_frame_size)
 
@@ -252,7 +251,6 @@ class Stream:
 
             self._conn.send_data(self.stream_id, data_chunk, end_stream=False)
 
-            data_frames_sent += 1
             bytes_to_send_size = bytes_to_send_size - chunk_size
             self.remaining_content_length = self.remaining_content_length - chunk_size
 
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index ca8a629b1..98dc98a0f 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -42,8 +42,8 @@ def make_html_body(val):
 
 
 class Data:
-    SMALL_SIZE = 1024 * 10  # 10 KB
-    LARGE_SIZE = (1024 ** 2) * 10  # 10 MB
+    SMALL_SIZE = 1024  # 1 KB
+    LARGE_SIZE = 1024 ** 2  # 1 MB
 
     STR_SMALL = generate_random_string(SMALL_SIZE)
     STR_LARGE = generate_random_string(LARGE_SIZE)

From 7f5bb6b34c6a5137aa12cd4ad4f3845a8c67653f Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Mon, 6 Jul 2020 13:08:14 +0530
Subject: [PATCH 070/716] chore: add h2 to setup.py, tox.ini

- Change log level for hpack to ERROR
---
 scrapy/utils/log.py | 3 +++
 setup.py            | 3 ++-
 tox.ini             | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 51d276097..4e2671478 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -52,6 +52,9 @@ DEFAULT_LOGGING = {
         'twisted': {
             'level': 'ERROR',
         },
+        'hpack': {
+            'level': 'ERROR',
+        },
     }
 }
 
diff --git a/setup.py b/setup.py
index 47c5906e4..d872d6472 100644
--- a/setup.py
+++ b/setup.py
@@ -81,7 +81,8 @@ setup(
         'zope.interface>=4.1.3',
         'protego>=0.1.15',
         'itemadapter>=0.1.0',
-        'typing_extensions>=3.7'
+        'typing_extensions>=3.7',
+        'h2>=3.2.0'
     ],
     extras_require=extras_require,
 )
\ No newline at end of file
diff --git a/tox.ini b/tox.ini
index ada211b3c..bc6314a2f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -84,6 +84,7 @@ deps =
     # Extras
     botocore==1.3.23
     Pillow==3.4.2
+    h2==3.2.0
 
 [testenv:extra-deps]
 deps =

From 54e4228c3a22164b79db36a29a5e2d64391b592a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Mon, 6 Jul 2020 14:10:45 -0300
Subject: [PATCH 071/716] refactor: use protocol

- H2ClientProtocol.close_stream
- Fix and add missing type hints
- More adjustments
- Rename stream id generator
- Simplify decrement
---
 scrapy/core/http2/protocol.py |  92 +++++++++++++++---------------
 scrapy/core/http2/stream.py   | 103 ++++++++++++++++------------------
 2 files changed, 93 insertions(+), 102 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index a3dfdb76e..2f177656d 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -11,32 +11,34 @@ from h2.events import (
     StreamEnded, StreamReset, WindowUpdated
 )
 from h2.exceptions import ProtocolError
+from twisted.internet.defer import Deferred
 from twisted.internet.protocol import connectionDone, Protocol
 from twisted.internet.ssl import Certificate
+from twisted.python.failure import Failure
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
 from scrapy.core.http2.types import H2ConnectionMetadataDict
 from scrapy.http import Request
 
+
 logger = logging.getLogger(__name__)
 
 
 class H2ClientProtocol(Protocol):
-    def __init__(self):
+    def __init__(self) -> None:
         config = H2Configuration(client_side=True, header_encoding='utf-8')
         self.conn = H2Connection(config=config)
 
         # ID of the next request stream
-        # Following the convention made by hyper-h2 each client ID
-        # will be odd.
-        self.stream_id_count = itertools.count(start=1, step=2)
+        # Following the convention made by hyper-h2 all IDs will be odd
+        self._stream_id_generator = itertools.count(start=1, step=2)
 
         # Streams are stored in a dictionary keyed off their stream IDs
         self.streams: Dict[int, Stream] = {}
 
         # If requests are received before connection is made we keep
         # all requests in a pool and send them as the connection is made
-        self._pending_request_stream_pool = deque()
+        self._pending_request_stream_pool: deque = deque()
 
         # Counter to keep track of opened stream. This counter
         # is used to make sure that not more than MAX_CONCURRENT_STREAMS
@@ -48,15 +50,15 @@ class H2ClientProtocol(Protocol):
         # We pass this instance to the streams ResponseFailed() failure
         self._protocol_error: Optional[ProtocolError] = None
 
-        self._metadata: H2ConnectionMetadataDict = {
+        self.metadata: H2ConnectionMetadataDict = {
             'certificate': None,
             'ip_address': None,
             'hostname': None,
-            'port': None
+            'port': None,
         }
 
     @property
-    def is_connected(self):
+    def is_connected(self) -> bool:
         """Boolean to keep track of the connection status.
         This is used while initiating pending streams to make sure
         that we initiate stream only during active HTTP/2 Connection
@@ -75,7 +77,7 @@ class H2ClientProtocol(Protocol):
             self.conn.remote_settings.max_concurrent_streams
         )
 
-    def _send_pending_requests(self):
+    def _send_pending_requests(self) -> None:
         """Initiate all pending requests from the deque following FIFO
         We make sure that at any time {allowed_max_concurrent_streams}
         streams are active.
@@ -89,37 +91,33 @@ class H2ClientProtocol(Protocol):
             stream = self._pending_request_stream_pool.popleft()
             stream.initiate_request()
 
-    def _stream_close_cb(self, stream_id: int):
-        """Called when stream is closed completely
+    def pop_stream(self, stream_id: int) -> Stream:
+        """Perform cleanup when a stream is closed
         """
-        self.streams.pop(stream_id)
+        stream = self.streams.pop(stream_id)
         self._active_streams -= 1
         self._send_pending_requests()
+        return stream
 
-    def _new_stream(self, request: Request):
+    def _new_stream(self, request: Request) -> Stream:
         """Instantiates a new Stream object
         """
-        stream_id = next(self.stream_id_count)
-
         stream = Stream(
-            stream_id=stream_id,
+            stream_id=next(self._stream_id_generator),
             request=request,
-            connection=self.conn,
-            conn_metadata=self._metadata,
-            cb_close=self._stream_close_cb
+            protocol=self,
         )
-
         self.streams[stream.stream_id] = stream
         return stream
 
-    def _write_to_transport(self):
+    def _write_to_transport(self) -> None:
         """ Write data to the underlying transport connection
         from the HTTP2 connection instance if any
         """
         data = self.conn.data_to_send()
         self.transport.write(data)
 
-    def request(self, request: Request):
+    def request(self, request: Request) -> Deferred:
         if not isinstance(request, Request):
             raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}')
 
@@ -130,20 +128,20 @@ class H2ClientProtocol(Protocol):
         self._pending_request_stream_pool.append(stream)
         return d
 
-    def connectionMade(self):
+    def connectionMade(self) -> None:
         """Called by Twisted when the connection is established. We can start
         sending some data now: we should open with the connection preamble.
         """
         destination = self.transport.getPeer()
         logger.debug('Connection made to {}'.format(destination))
-        self._metadata['ip_address'] = ipaddress.ip_address(destination.host)
-        self._metadata['port'] = destination.port
-        self._metadata['hostname'] = self.transport.transport.addr[0]
+        self.metadata['ip_address'] = ipaddress.ip_address(destination.host)
+        self.metadata['port'] = destination.port
+        self.metadata['hostname'] = self.transport.transport.addr[0]
 
         self.conn.initiate_connection()
         self._write_to_transport()
 
-    def dataReceived(self, data):
+    def dataReceived(self, data: bytes) -> None:
         try:
             events = self.conn.receive_data(data)
             self._handle_events(events)
@@ -158,32 +156,30 @@ class H2ClientProtocol(Protocol):
         finally:
             self._write_to_transport()
 
-    def connectionLost(self, reason=connectionDone):
+    def connectionLost(self, reason: Failure = connectionDone) -> None:
         """Called by Twisted when the transport connection is lost.
         No need to write anything to transport here.
         """
-        # Pop all streams which were pending and were not yet started
-        # NOTE: Stream.close() pops the element from the streams dictionary
-        # which raises `RuntimeError: dictionary changed size during iteration`
-        # Hence, we copy the streams into a list.
-        for stream in list(self.streams.values()):
+        for stream in self.streams.values():
             if stream.request_sent:
-                stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error)
+                stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error, from_protocol=True)
             else:
-                stream.close(StreamCloseReason.INACTIVE)
+                stream.close(StreamCloseReason.INACTIVE, from_protocol=True)
 
+        self._active_streams -= len(self.streams)
+        self.streams.clear()
+        self._send_pending_requests()
         self.conn.close_connection()
 
         if not reason.check(connectionDone):
             logger.warning("Connection lost with reason " + str(reason))
 
-    def _handle_events(self, events):
+    def _handle_events(self, events: list) -> None:
         """Private method which acts as a bridge between the events
         received from the HTTP/2 data and IH2EventsHandler
 
         Arguments:
-            events {list} -- A list of events that the remote peer
-                triggered by sending data
+            events -- A list of events that the remote peer triggered by sending data
         """
         for event in events:
             if isinstance(event, DataReceived):
@@ -202,27 +198,29 @@ class H2ClientProtocol(Protocol):
                 logger.debug('Received unhandled event {}'.format(event))
 
     # Event handler functions starts here
-    def data_received(self, event: DataReceived):
+    def data_received(self, event: DataReceived) -> None:
         self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length)
 
-    def response_received(self, event: ResponseReceived):
+    def response_received(self, event: ResponseReceived) -> None:
         self.streams[event.stream_id].receive_headers(event.headers)
 
-    def settings_acknowledged(self, event: SettingsAcknowledged):
+    def settings_acknowledged(self, event: SettingsAcknowledged) -> None:
         # Send off all the pending requests as now we have
         # established a proper HTTP/2 connection
         self._send_pending_requests()
 
         # Update certificate when our HTTP/2 connection is established
-        self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
+        self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
 
-    def stream_ended(self, event: StreamEnded):
-        self.streams[event.stream_id].close(StreamCloseReason.ENDED)
+    def stream_ended(self, event: StreamEnded) -> None:
+        stream = self.pop_stream(event.stream_id)
+        stream.close(StreamCloseReason.ENDED, from_protocol=True)
 
-    def stream_reset(self, event: StreamReset):
-        self.streams[event.stream_id].close(StreamCloseReason.RESET)
+    def stream_reset(self, event: StreamReset) -> None:
+        stream = self.pop_stream(event.stream_id)
+        stream.close(StreamCloseReason.RESET, from_protocol=True)
 
-    def window_updated(self, event: WindowUpdated):
+    def window_updated(self, event: WindowUpdated) -> None:
         if event.stream_id != 0:
             self.streams[event.stream_id].receive_window_update()
         else:
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 1c856ff68..77cfbcfbf 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -1,22 +1,27 @@
 import logging
 from enum import Enum
 from io import BytesIO
-from typing import Callable, List
+from typing import List, Optional, Tuple, TYPE_CHECKING
 from urllib.parse import urlparse
 
-from h2.connection import H2Connection
 from h2.errors import ErrorCodes
 from h2.exceptions import StreamClosedError
+from hpack import HeaderTuple
 from twisted.internet.defer import Deferred, CancelledError
 from twisted.internet.error import ConnectionClosed
 from twisted.python.failure import Failure
 from twisted.web.client import ResponseFailed
 
-from scrapy.core.http2.types import H2ConnectionMetadataDict, H2ResponseDict
+from scrapy.core.http2.types import H2ResponseDict
 from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
 
+
+if TYPE_CHECKING:
+    from scrapy.core.http2.protocol import H2ClientProtocol
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -31,12 +36,12 @@ class InactiveStreamClosed(ConnectionClosed):
 
 class InvalidHostname(Exception):
 
-    def __init__(self, request: Request, expected_hostname, expected_netloc):
+    def __init__(self, request: Request, expected_hostname: Optional[str], expected_netloc: Optional[str]) -> None:
         self.request = request
         self.expected_hostname = expected_hostname
         self.expected_netloc = expected_netloc
 
-    def __str__(self):
+    def __str__(self) -> str:
         return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}'
 
 
@@ -80,28 +85,20 @@ class Stream:
         self,
         stream_id: int,
         request: Request,
-        connection: H2Connection,
-        conn_metadata: H2ConnectionMetadataDict,
-        cb_close: Callable[[int], None],
+        protocol: "H2ClientProtocol",
         download_maxsize: int = 0,
         download_warnsize: int = 0,
         fail_on_data_loss: bool = True
-    ):
+    ) -> None:
         """
         Arguments:
-            stream_id -- For one HTTP/2 connection each stream is
-                uniquely identified by a single integer
-            request -- HTTP request
-            connection -- HTTP/2 connection this stream belongs to.
-            conn_metadata -- Reference to dictionary having metadata of HTTP/2 connection
-            cb_close -- Method called when this stream is closed
-                to notify the TCP connection instance.
+            stream_id -- Unique identifier for the stream within a single HTTP/2 connection
+            request -- The HTTP request associated to the stream
+            protocol -- Parent H2ClientProtocol instance
         """
-        self.stream_id = stream_id
-        self._request = request
-        self._conn = connection
-        self._conn_metadata = conn_metadata
-        self._cb_close = cb_close
+        self.stream_id: int = stream_id
+        self._request: Request = request
+        self._protocol: "H2ClientProtocol" = protocol
 
         self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
         self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
@@ -132,7 +129,7 @@ class Stream:
         self._response: H2ResponseDict = {
             'body': BytesIO(),
             'flow_controlled_size': 0,
-            'headers': Headers({})
+            'headers': Headers({}),
         }
 
         def _cancel(_):
@@ -145,7 +142,7 @@ class Stream:
 
         self._deferred_response = Deferred(_cancel)
 
-    def __str__(self):
+    def __str__(self) -> str:
         return f'Stream(id={self.stream_id!r})'
 
     __repr__ = __str__
@@ -169,12 +166,9 @@ class Stream:
             and not self._reached_warnsize
         )
 
-    def get_response(self):
+    def get_response(self) -> Deferred:
         """Simply return a Deferred which fires when response
         from the asynchronous request is available
-
-        Returns:
-            Deferred -- Calls the callback passing the response
         """
         return self._deferred_response
 
@@ -182,12 +176,12 @@ class Stream:
         # Make sure that we are sending the request to the correct URL
         url = urlparse(self._request.url)
         return (
-            url.netloc == self._conn_metadata['hostname']
-            or url.netloc == f'{self._conn_metadata["hostname"]}:{self._conn_metadata["port"]}'
-            or url.netloc == f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}'
+            url.netloc == self._protocol.metadata['hostname']
+            or url.netloc == f'{self._protocol.metadata["hostname"]}:{self._protocol.metadata["port"]}'
+            or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}'
         )
 
-    def _get_request_headers(self):
+    def _get_request_headers(self) -> List[Tuple[str, str]]:
         url = urlparse(self._request.url)
 
         path = url.path
@@ -207,10 +201,10 @@ class Stream:
 
         return headers
 
-    def initiate_request(self):
+    def initiate_request(self) -> None:
         if self.check_request_url():
             headers = self._get_request_headers()
-            self._conn.send_headers(self.stream_id, headers, end_stream=False)
+            self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False)
             self.request_sent = True
             self.send_data()
         else:
@@ -218,7 +212,7 @@ class Stream:
             # Note that we have not sent any headers
             self.close(StreamCloseReason.INVALID_HOSTNAME)
 
-    def send_data(self):
+    def send_data(self) -> None:
         """Called immediately after the headers are sent. Here we send all the
          data as part of the request.
 
@@ -233,10 +227,10 @@ class Stream:
             raise StreamClosedError(self.stream_id)
 
         # Firstly, check what the flow control window is for current stream.
-        window_size = self._conn.local_flow_control_window(stream_id=self.stream_id)
+        window_size = self._protocol.conn.local_flow_control_window(stream_id=self.stream_id)
 
         # Next, check what the maximum frame size is.
-        max_frame_size = self._conn.max_outbound_frame_size
+        max_frame_size = self._protocol.conn.max_outbound_frame_size
 
         # We will send no more than the window size or the remaining file size
         # of data in this call, whichever is smaller.
@@ -249,7 +243,7 @@ class Stream:
             data_chunk_start_id = self.content_length - self.remaining_content_length
             data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size]
 
-            self._conn.send_data(self.stream_id, data_chunk, end_stream=False)
+            self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False)
 
             bytes_to_send_size = bytes_to_send_size - chunk_size
             self.remaining_content_length = self.remaining_content_length - chunk_size
@@ -258,12 +252,12 @@ class Stream:
 
         # End the stream if no more data needs to be send
         if self.remaining_content_length == 0:
-            self._conn.end_stream(self.stream_id)
+            self._protocol.conn.end_stream(self.stream_id)
 
         # Q. What about the rest of the data?
         # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame
 
-    def receive_window_update(self):
+    def receive_window_update(self) -> None:
         """Flow control window size was changed.
         Send data that earlier could not be sent as we were
         blocked behind the flow control.
@@ -271,7 +265,7 @@ class Stream:
         if self.remaining_content_length and not self.stream_closed_server and self.request_sent:
             self.send_data()
 
-    def receive_data(self, data: bytes, flow_controlled_length: int):
+    def receive_data(self, data: bytes, flow_controlled_length: int) -> None:
         self._response['body'].write(data)
         self._response['flow_controlled_size'] += flow_controlled_length
 
@@ -289,12 +283,12 @@ class Stream:
             logger.warning(warning_msg)
 
         # Acknowledge the data received
-        self._conn.acknowledge_received_data(
+        self._protocol.conn.acknowledge_received_data(
             self._response['flow_controlled_size'],
             self.stream_id
         )
 
-    def receive_headers(self, headers):
+    def receive_headers(self, headers: List[HeaderTuple]) -> None:
         for name, value in headers:
             self._response['headers'][name] = value
 
@@ -312,7 +306,7 @@ class Stream:
             )
             logger.warning(warning_msg)
 
-    def reset_stream(self, reason=StreamCloseReason.RESET):
+    def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None:
         """Close this stream by sending a RST_FRAME to the remote peer"""
         if self.stream_closed_local:
             raise StreamClosedError(self.stream_id)
@@ -321,7 +315,7 @@ class Stream:
         self._response['body'].truncate(0)
 
         self.stream_closed_local = True
-        self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
+        self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
         self.close(reason)
 
     def _is_data_lost(self) -> bool:
@@ -332,11 +326,8 @@ class Stream:
 
         return expected_size != received_body_size
 
-    def close(self, reason: StreamCloseReason, error: Exception = None):
+    def close(self, reason: StreamCloseReason, error: Optional[Exception] = None, from_protocol: bool = False) -> None:
         """Based on the reason sent we will handle each case.
-
-        Arguments:
-            reason -- One if StreamCloseReason
         """
         if self.stream_closed_server:
             raise StreamClosedError(self.stream_id)
@@ -344,7 +335,9 @@ class Stream:
         if not isinstance(reason, StreamCloseReason):
             raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}')
 
-        self._cb_close(self.stream_id)
+        if not from_protocol:
+            self._protocol.pop_stream(self.stream_id)
+
         self.stream_closed_server = True
 
         flags = None
@@ -392,11 +385,11 @@ class Stream:
         elif reason is StreamCloseReason.INVALID_HOSTNAME:
             self._deferred_response.errback(InvalidHostname(
                 self._request,
-                self._conn_metadata['hostname'],
-                f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}'
+                self._protocol.metadata['hostname'],
+                f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}'
             ))
 
-    def _fire_response_deferred(self, flags: List[str] = None):
+    def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None:
         """Builds response from the self._response dict
         and fires the response deferred callback with the
         generated response instance"""
@@ -405,7 +398,7 @@ class Stream:
         response_cls = responsetypes.from_args(
             headers=self._response['headers'],
             url=self._request.url,
-            body=body
+            body=body,
         )
 
         response = response_cls(
@@ -415,8 +408,8 @@ class Stream:
             body=body,
             request=self._request,
             flags=flags,
-            certificate=self._conn_metadata['certificate'],
-            ip_address=self._conn_metadata['ip_address']
+            certificate=self._protocol.metadata['certificate'],
+            ip_address=self._protocol.metadata['ip_address'],
         )
 
         self._deferred_response.callback(response)

From 1c40dfa7408026bc9ae831000a8614e8f4a9dd0d Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Tue, 7 Jul 2020 00:44:09 +0530
Subject: [PATCH 072/716] fix: handle CONNECTION_LOST & RESET separately

---
 scrapy/core/http2/protocol.py       | 19 ++++++++++++-------
 scrapy/core/http2/stream.py         | 18 ++++++++++++------
 scrapy/utils/log.py                 |  6 +++---
 tests/test_http2_client_protocol.py |  2 +-
 tox.ini                             |  2 +-
 5 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 2f177656d..55dbcabec 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -20,7 +20,6 @@ from scrapy.core.http2.stream import Stream, StreamCloseReason
 from scrapy.core.http2.types import H2ConnectionMetadataDict
 from scrapy.http import Request
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -30,7 +29,8 @@ class H2ClientProtocol(Protocol):
         self.conn = H2Connection(config=config)
 
         # ID of the next request stream
-        # Following the convention made by hyper-h2 all IDs will be odd
+        # Following the convention - 'Streams initiated by a client MUST
+        # use odd-numbered stream identifiers' (RFC 7540)
         self._stream_id_generator = itertools.count(start=1, step=2)
 
         # Streams are stored in a dictionary keyed off their stream IDs
@@ -160,20 +160,25 @@ class H2ClientProtocol(Protocol):
         """Called by Twisted when the transport connection is lost.
         No need to write anything to transport here.
         """
+        errors = []
+        if not reason.check(connectionDone):
+            logger.warning("Connection lost with reason " + str(reason))
+            errors.append(reason)
+
+        if self._protocol_error:
+            errors.append(self._protocol_error)
+
         for stream in self.streams.values():
             if stream.request_sent:
-                stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error, from_protocol=True)
+                stream.close(StreamCloseReason.CONNECTION_LOST, errors, from_protocol=True)
             else:
                 stream.close(StreamCloseReason.INACTIVE, from_protocol=True)
 
         self._active_streams -= len(self.streams)
         self.streams.clear()
-        self._send_pending_requests()
+        self._pending_request_stream_pool.clear()
         self.conn.close_connection()
 
-        if not reason.check(connectionDone):
-            logger.warning("Connection lost with reason " + str(reason))
-
     def _handle_events(self, events: list) -> None:
         """Private method which acts as a bridge between the events
         received from the HTTP/2 data and IH2EventsHandler
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 77cfbcfbf..8b66d4b85 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -17,11 +17,9 @@ from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
 
-
 if TYPE_CHECKING:
     from scrapy.core.http2.protocol import H2ClientProtocol
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -326,7 +324,12 @@ class Stream:
 
         return expected_size != received_body_size
 
-    def close(self, reason: StreamCloseReason, error: Optional[Exception] = None, from_protocol: bool = False) -> None:
+    def close(
+        self,
+        reason: StreamCloseReason,
+        errors: Optional[List[Exception]] = None,
+        from_protocol: bool = False
+    ) -> None:
         """Based on the reason sent we will handle each case.
         """
         if self.stream_closed_server:
@@ -374,11 +377,14 @@ class Stream:
             self._response['headers'][':status'] = '499'
             self._fire_response_deferred()
 
-        elif reason in (StreamCloseReason.RESET, StreamCloseReason.CONNECTION_LOST):
+        elif reason is StreamCloseReason.RESET:
             self._deferred_response.errback(ResponseFailed([
-                error if error else Failure()
+                Failure(f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM')
             ]))
 
+        elif reason is StreamCloseReason.CONNECTION_LOST:
+            self._deferred_response.errback(ResponseFailed(errors))
+
         elif reason is StreamCloseReason.INACTIVE:
             self._deferred_response.errback(InactiveStreamClosed(self._request))
 
@@ -403,7 +409,7 @@ class Stream:
 
         response = response_cls(
             url=self._request.url,
-            status=self._response['headers'][':status'],
+            status=int(self._response['headers'][':status']),
             headers=self._response['headers'],
             body=body,
             request=self._request,
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 4e2671478..9d59fdd68 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -46,15 +46,15 @@ DEFAULT_LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
     'loggers': {
+        'hpack': {
+            'level': 'ERROR',
+        },
         'scrapy': {
             'level': 'DEBUG',
         },
         'twisted': {
             'level': 'ERROR',
         },
-        'hpack': {
-            'level': 'ERROR',
-        },
     }
 }
 
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 98dc98a0f..9efca5267 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -474,7 +474,7 @@ class Https2ClientProtocolTestCase(TestCase):
 
         # Close the connection now to fire all the extra 10 requests errback
         # with InactiveStreamClosed
-        self.client.transport.abortConnection()
+        self.client.transport.loseConnection()
 
         return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True)
 
diff --git a/tox.ini b/tox.ini
index bc6314a2f..be3b56e2d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -83,8 +83,8 @@ deps =
     -rtests/requirements-py3.txt
     # Extras
     botocore==1.3.23
-    Pillow==3.4.2
     h2==3.2.0
+    Pillow==3.4.2
 
 [testenv:extra-deps]
 deps =

From 2ea7d82534cafe5b25b28ef5a78e5b714767d27a Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 8 Jul 2020 18:57:13 +0530
Subject: [PATCH 073/716] feat: H2ClientFactory

---
 scrapy/core/http2/protocol.py       | 35 +++++++++++++++++++++--------
 scrapy/core/http2/stream.py         | 29 +++++++++++++-----------
 scrapy/core/http2/types.py          | 13 ++++++-----
 tests/test_http2_client_protocol.py |  8 ++++---
 4 files changed, 55 insertions(+), 30 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 55dbcabec..ee51300cc 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -2,29 +2,38 @@ import ipaddress
 import itertools
 import logging
 from collections import deque
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
 from h2.events import (
-    DataReceived, ResponseReceived, SettingsAcknowledged,
+    Event, DataReceived, ResponseReceived, SettingsAcknowledged,
     StreamEnded, StreamReset, WindowUpdated
 )
 from h2.exceptions import ProtocolError
 from twisted.internet.defer import Deferred
-from twisted.internet.protocol import connectionDone, Protocol
+from twisted.internet.protocol import connectionDone, Factory, Protocol
 from twisted.internet.ssl import Certificate
 from twisted.python.failure import Failure
+from twisted.web.client import URI
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
 from scrapy.core.http2.types import H2ConnectionMetadataDict
 from scrapy.http import Request
+from scrapy.settings import Settings
 
 logger = logging.getLogger(__name__)
 
 
 class H2ClientProtocol(Protocol):
-    def __init__(self) -> None:
+    def __init__(self, uri: URI, settings: Settings) -> None:
+        """
+        Arguments:
+            uri -- URI of the base url to which HTTP/2 Connection will be made.
+                uri is used to verify that incoming client requests have correct
+                base URL.
+            settings -- Scrapy project settings
+        """
         config = H2Configuration(client_side=True, header_encoding='utf-8')
         self.conn = H2Connection(config=config)
 
@@ -53,8 +62,9 @@ class H2ClientProtocol(Protocol):
         self.metadata: H2ConnectionMetadataDict = {
             'certificate': None,
             'ip_address': None,
-            'hostname': None,
-            'port': None,
+            'uri': uri,
+            'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'),
+            'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'),
         }
 
     @property
@@ -135,8 +145,6 @@ class H2ClientProtocol(Protocol):
         destination = self.transport.getPeer()
         logger.debug('Connection made to {}'.format(destination))
         self.metadata['ip_address'] = ipaddress.ip_address(destination.host)
-        self.metadata['port'] = destination.port
-        self.metadata['hostname'] = self.transport.transport.addr[0]
 
         self.conn.initiate_connection()
         self._write_to_transport()
@@ -179,7 +187,7 @@ class H2ClientProtocol(Protocol):
         self._pending_request_stream_pool.clear()
         self.conn.close_connection()
 
-    def _handle_events(self, events: list) -> None:
+    def _handle_events(self, events: List[Event]) -> None:
         """Private method which acts as a bridge between the events
         received from the HTTP/2 data and IH2EventsHandler
 
@@ -232,3 +240,12 @@ class H2ClientProtocol(Protocol):
             # Send leftover data for all the streams
             for stream in self.streams.values():
                 stream.receive_window_update()
+
+
+class H2ClientFactory(Factory):
+    def __init__(self, uri: URI, settings: Settings) -> None:
+        self.uri = uri
+        self.settings = settings
+
+    def buildProtocol(self, addr) -> H2ClientProtocol:
+        return H2ClientProtocol(self.uri, self.settings)
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 8b66d4b85..5017f9cd4 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -16,6 +16,7 @@ from scrapy.core.http2.types import H2ResponseDict
 from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
+from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
     from scrapy.core.http2.protocol import H2ClientProtocol
@@ -34,7 +35,7 @@ class InactiveStreamClosed(ConnectionClosed):
 
 class InvalidHostname(Exception):
 
-    def __init__(self, request: Request, expected_hostname: Optional[str], expected_netloc: Optional[str]) -> None:
+    def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None:
         self.request = request
         self.expected_hostname = expected_hostname
         self.expected_netloc = expected_netloc
@@ -83,10 +84,7 @@ class Stream:
         self,
         stream_id: int,
         request: Request,
-        protocol: "H2ClientProtocol",
-        download_maxsize: int = 0,
-        download_warnsize: int = 0,
-        fail_on_data_loss: bool = True
+        protocol: "H2ClientProtocol"
     ) -> None:
         """
         Arguments:
@@ -98,9 +96,14 @@ class Stream:
         self._request: Request = request
         self._protocol: "H2ClientProtocol" = protocol
 
-        self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
-        self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
-        self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss)
+        self._download_maxsize = self._request.meta.get(
+            'download_maxsize',
+            self._protocol.metadata['default_download_maxsize']
+        )
+        self._download_warnsize = self._request.meta.get(
+            'download_warnsize',
+            self._protocol.metadata['default_download_warnsize']
+        )
 
         self.request_start_time = None
 
@@ -174,9 +177,9 @@ class Stream:
         # Make sure that we are sending the request to the correct URL
         url = urlparse(self._request.url)
         return (
-            url.netloc == self._protocol.metadata['hostname']
-            or url.netloc == f'{self._protocol.metadata["hostname"]}:{self._protocol.metadata["port"]}'
-            or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}'
+            url.netloc == to_unicode(self._protocol.metadata['uri'].host)
+            or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc)
+            or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
         )
 
     def _get_request_headers(self) -> List[Tuple[str, str]]:
@@ -391,8 +394,8 @@ class Stream:
         elif reason is StreamCloseReason.INVALID_HOSTNAME:
             self._deferred_response.errback(InvalidHostname(
                 self._request,
-                self._protocol.metadata['hostname'],
-                f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}'
+                to_unicode(self._protocol.metadata['uri'].host),
+                f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
             ))
 
     def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None:
diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py
index dd7b1187b..d2aa1a9d8 100644
--- a/scrapy/core/http2/types.py
+++ b/scrapy/core/http2/types.py
@@ -3,6 +3,7 @@ from ipaddress import IPv4Address, IPv6Address
 from typing import Union, Optional
 
 from twisted.internet.ssl import Certificate
+from twisted.web.client import URI
 # for python < 3.8 -- typing.TypedDict is undefined
 from typing_extensions import TypedDict
 
@@ -19,14 +20,16 @@ class H2ConnectionMetadataDict(TypedDict):
     # is updated when HTTP/2 connection is  made successfully
     ip_address: Optional[Union[IPv4Address, IPv6Address]]
 
-    # Name of the peer HTTP/2 connection is established
-    hostname: Optional[str]
+    # URI of the peer HTTP/2 connection is made
+    uri: URI
 
-    port: Optional[int]
-
-    # Both ip_address and hostname are used by the Stream before
+    # Both ip_address and uri are used by the Stream before
     # initiating the request to verify that the base address
 
+    # Variables taken from Project Settings
+    default_download_maxsize: int
+    default_download_warnsize: int
+
 
 class H2ResponseDict(TypedDict):
     # Data received frame by frame from the server is appended
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 9efca5267..05e5f5047 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -11,17 +11,18 @@ from h2.exceptions import InvalidBodyLengthError
 from twisted.internet import reactor
 from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
-from twisted.internet.protocol import Factory
 from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate
 from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
+from twisted.web.client import URI
 from twisted.web.http import Request as TxRequest
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.static import File
 
-from scrapy.core.http2.protocol import H2ClientProtocol
+from scrapy.core.http2.protocol import H2ClientFactory
 from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
 from scrapy.http import Request, Response, JsonRequest
+from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes, to_unicode
 from tests.mockserver import ssl_context_factory, LeafResource, Status
 
@@ -183,7 +184,8 @@ class Https2ClientProtocolTestCase(TestCase):
             trustRoot=self.client_certificate,
             acceptableProtocols=[b'h2']
         )
-        h2_client_factory = Factory.forProtocol(H2ClientProtocol)
+        uri = URI.fromBytes(to_bytes(self.get_url('/')))
+        h2_client_factory = H2ClientFactory(uri, Settings())
         client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
         self.client = yield client_endpoint.connect(h2_client_factory)
 

From 64c6af10e1b276db68ea722845621912d2023a75 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Mon, 13 Jul 2020 00:57:49 +0530
Subject: [PATCH 074/716] refactor: use str instead of to_unicode

---
 scrapy/core/http2/stream.py         |  7 +++----
 tests/test_http2_client_protocol.py | 29 ++++++++++++++++-------------
 2 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 5017f9cd4..a2f0e2aa4 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -16,7 +16,6 @@ from scrapy.core.http2.types import H2ResponseDict
 from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
-from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
     from scrapy.core.http2.protocol import H2ClientProtocol
@@ -177,8 +176,8 @@ class Stream:
         # Make sure that we are sending the request to the correct URL
         url = urlparse(self._request.url)
         return (
-            url.netloc == to_unicode(self._protocol.metadata['uri'].host)
-            or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc)
+            url.netloc == str(self._protocol.metadata['uri'].host, 'utf-8')
+            or url.netloc == str(self._protocol.metadata['uri'].netloc, 'utf-8')
             or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
         )
 
@@ -394,7 +393,7 @@ class Stream:
         elif reason is StreamCloseReason.INVALID_HOSTNAME:
             self._deferred_response.errback(InvalidHostname(
                 self._request,
-                to_unicode(self._protocol.metadata['uri'].host),
+                str(self._protocol.metadata['uri'].host, 'utf-8'),
                 f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
             ))
 
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 05e5f5047..05f4889ef 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -23,7 +23,6 @@ from scrapy.core.http2.protocol import H2ClientFactory
 from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
 from scrapy.http import Request, Response, JsonRequest
 from scrapy.settings import Settings
-from scrapy.utils.python import to_bytes, to_unicode
 from tests.mockserver import ssl_context_factory, LeafResource, Status
 
 
@@ -39,7 +38,7 @@ def make_html_body(val):
 <h1>Hello from HTTP2<h1>
 <p>{val}</p>
 </html>'''
-    return to_bytes(response)
+    return bytes(response, 'utf-8')
 
 
 class Data:
@@ -83,10 +82,11 @@ class PostDataJsonMixin:
             'extra-data': extra_data
         }
         for k, v in request.requestHeaders.getAllRawHeaders():
-            response['request-headers'][to_unicode(k)] = to_unicode(v[0])
+            response['request-headers'][str(k, 'utf-8')] = str(v[0], 'utf-8')
 
-        response_bytes = to_bytes(json.dumps(response))
-        request.setHeader('Content-Type', 'application/json')
+        response_bytes = bytes(json.dumps(response), 'utf-8')
+        request.setHeader('Content-Type', 'application/json; charset=UTF-8')
+        request.setHeader('Content-Encoding', 'UTF-8')
         return response_bytes
 
 
@@ -127,13 +127,14 @@ class NoContentLengthHeader(LeafResource):
 
 class QueryParams(LeafResource):
     def render_GET(self, request: TxRequest):
-        request.setHeader('Content-Type', 'application/json')
+        request.setHeader('Content-Type', 'application/json; charset=UTF-8')
+        request.setHeader('Content-Encoding', 'UTF-8')
 
         query_params = {}
         for k, v in request.args.items():
-            query_params[to_unicode(k)] = to_unicode(v[0])
+            query_params[str(k, 'utf-8')] = str(v[0], 'utf-8')
 
-        return to_bytes(json.dumps(query_params))
+        return bytes(json.dumps(query_params), 'utf-8')
 
 
 def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
@@ -184,7 +185,7 @@ class Https2ClientProtocolTestCase(TestCase):
             trustRoot=self.client_certificate,
             acceptableProtocols=[b'h2']
         )
-        uri = URI.fromBytes(to_bytes(self.get_url('/')))
+        uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8'))
         h2_client_factory = H2ClientFactory(uri, Settings())
         client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
         self.client = yield client_endpoint.connect(h2_client_factory)
@@ -278,7 +279,8 @@ class Https2ClientProtocolTestCase(TestCase):
             self.assertEqual(len(response.body), content_length)
 
             # Parse the body
-            body = json.loads(to_unicode(response.body))
+            content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8')
+            body = json.loads(str(response.body, content_encoding))
             self.assertIn('request-body', body)
             self.assertIn('extra-data', body)
             self.assertIn('request-headers', body)
@@ -292,9 +294,9 @@ class Https2ClientProtocolTestCase(TestCase):
             # Check if headers were sent successfully
             request_headers = body['request-headers']
             for k, v in request.headers.items():
-                k_str = to_unicode(k)
+                k_str = str(k, 'utf-8')
                 self.assertIn(k_str, request_headers)
-                self.assertEqual(request_headers[k_str], to_unicode(v[0]))
+                self.assertEqual(request_headers[k_str], str(v[0], 'utf-8'))
 
         d.addCallback(assert_response)
         d.addErrback(self.fail)
@@ -494,7 +496,8 @@ class Https2ClientProtocolTestCase(TestCase):
         request = Request(self.get_url(f'/query-params?{urlencode(params)}'))
 
         def assert_query_params(response: Response):
-            data = json.loads(to_unicode(response.body))
+            content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8')
+            data = json.loads(str(response.body, content_encoding))
             self.assertEqual(data, params)
 
         d = self.client.request(request)

From 0770961054f24d56d219a81d9e0c467de98312c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 13 Jul 2020 16:05:57 +0200
Subject: [PATCH 075/716] Write a test for #4665

---
 tests/test_commands.py | 37 ++++++++++++++++++++++++++++++++++---
 1 file changed, 34 insertions(+), 3 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 002237824..2e5bd6c00 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -66,9 +66,14 @@ class ProjectTest(unittest.TestCase):
 
     def proc(self, *new_args, **popen_kwargs):
         args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
-        p = subprocess.Popen(args, cwd=self.cwd, env=self.env,
-                             stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                             **popen_kwargs)
+        p = subprocess.Popen(
+            args,
+            cwd=popen_kwargs.pop('cwd', self.cwd),
+            env=self.env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            **popen_kwargs,
+        )
 
         def kill_proc():
             p.kill()
@@ -122,6 +127,32 @@ class StartprojectTest(ProjectTest):
         self.assertEqual(2, self.call('startproject'))
         self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params'))
 
+    def test_existing_project_dir(self):
+        project_dir = mkdtemp()
+        os.mkdir(os.path.join(project_dir, self.project_name))
+
+        p, out, err = self.proc('startproject', self.project_name, cwd=project_dir)
+        print(out)
+        print(err, file=sys.stderr)
+        self.assertEqual(p.returncode, 0)
+
+        assert exists(join(abspath(project_dir), 'scrapy.cfg'))
+        assert exists(join(abspath(project_dir), 'testproject'))
+        assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py'))
+        assert exists(join(join(abspath(project_dir), self.project_name), 'items.py'))
+        assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py'))
+        assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py'))
+        assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py'))
+
+        self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2'))
+
+        self.assertEqual(1, self.call('startproject', self.project_name, project_dir))
+        self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir))
+        self.assertEqual(1, self.call('startproject', 'wrong---project---name'))
+        self.assertEqual(1, self.call('startproject', 'sys'))
+        self.assertEqual(2, self.call('startproject'))
+        self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params'))
+
 
 def get_permissions_dict(path, renamings=None, ignore=None):
     renamings = renamings or tuple()

From 544c1f6e390c72053f768d3992ea2d0801363b83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 13 Jul 2020 16:30:34 +0200
Subject: [PATCH 076/716] Fix the issue

---
 scrapy/commands/startproject.py |  8 ++++----
 tests/test_commands.py          | 34 +++++++++++++++------------------
 2 files changed, 19 insertions(+), 23 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index e5158d993..35b58090c 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,7 +1,7 @@
 import re
 import os
 import string
-from importlib import import_module
+from importlib.util import find_spec
 from os.path import join, exists, abspath
 from shutil import ignore_patterns, move, copy2, copystat
 from stat import S_IWUSR as OWNER_WRITE_PERMISSION
@@ -43,10 +43,10 @@ class Command(ScrapyCommand):
     def _is_valid_name(self, project_name):
         def _module_exists(module_name):
             try:
-                import_module(module_name)
-                return True
-            except ImportError:
+                spec = find_spec(module_name)
+            except ModuleNotFoundError:
                 return False
+            return spec is not None and spec.loader is not None
 
         if not re.search(r'^[_a-zA-Z]\w*$', project_name):
             print('Error: Project names must begin with a letter and contain'
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 2e5bd6c00..10a3aa16c 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -92,7 +92,10 @@ class ProjectTest(unittest.TestCase):
 class StartprojectTest(ProjectTest):
 
     def test_startproject(self):
-        self.assertEqual(0, self.call('startproject', self.project_name))
+        p, out, err = self.proc('startproject', self.project_name)
+        print(out)
+        print(err, file=sys.stderr)
+        self.assertEqual(p.returncode, 0)
 
         assert exists(join(self.proj_path, 'scrapy.cfg'))
         assert exists(join(self.proj_path, 'testproject'))
@@ -129,29 +132,22 @@ class StartprojectTest(ProjectTest):
 
     def test_existing_project_dir(self):
         project_dir = mkdtemp()
-        os.mkdir(os.path.join(project_dir, self.project_name))
+        project_name = self.project_name + '_existing'
+        project_path = os.path.join(project_dir, project_name)
+        os.mkdir(project_path)
 
-        p, out, err = self.proc('startproject', self.project_name, cwd=project_dir)
+        p, out, err = self.proc('startproject', project_name, cwd=project_dir)
         print(out)
         print(err, file=sys.stderr)
         self.assertEqual(p.returncode, 0)
 
-        assert exists(join(abspath(project_dir), 'scrapy.cfg'))
-        assert exists(join(abspath(project_dir), 'testproject'))
-        assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py'))
-        assert exists(join(join(abspath(project_dir), self.project_name), 'items.py'))
-        assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py'))
-        assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py'))
-        assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py'))
-
-        self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2'))
-
-        self.assertEqual(1, self.call('startproject', self.project_name, project_dir))
-        self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir))
-        self.assertEqual(1, self.call('startproject', 'wrong---project---name'))
-        self.assertEqual(1, self.call('startproject', 'sys'))
-        self.assertEqual(2, self.call('startproject'))
-        self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params'))
+        assert exists(join(abspath(project_path), 'scrapy.cfg'))
+        assert exists(join(abspath(project_path), project_name))
+        assert exists(join(join(abspath(project_path), project_name), '__init__.py'))
+        assert exists(join(join(abspath(project_path), project_name), 'items.py'))
+        assert exists(join(join(abspath(project_path), project_name), 'pipelines.py'))
+        assert exists(join(join(abspath(project_path), project_name), 'settings.py'))
+        assert exists(join(join(abspath(project_path), project_name), 'spiders', '__init__.py'))
 
 
 def get_permissions_dict(path, renamings=None, ignore=None):

From aeaeb7385b7d1e6570bad38da4db492de8fb4206 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Tue, 14 Jul 2020 03:55:14 +0530
Subject: [PATCH 077/716] feat: assert negotiated protocol as h2

- implement IHandshakeListener in H2ClientProtocol to know when
handshake is completed
- implement IProtocolNegotiationFactory in H2ClientFactory to provide
information about the acceptableProtols (h2) during NPN or ALPN protocol
---
 scrapy/core/http2/protocol.py       | 69 ++++++++++++++++++++++-------
 scrapy/core/http2/stream.py         |  4 +-
 tests/test_http2_client_protocol.py |  2 +-
 3 files changed, 56 insertions(+), 19 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index ee51300cc..5d859d475 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -12,10 +12,12 @@ from h2.events import (
 )
 from h2.exceptions import ProtocolError
 from twisted.internet.defer import Deferred
+from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
 from twisted.internet.protocol import connectionDone, Factory, Protocol
 from twisted.internet.ssl import Certificate
 from twisted.python.failure import Failure
 from twisted.web.client import URI
+from zope.interface import implementer
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
 from scrapy.core.http2.types import H2ConnectionMetadataDict
@@ -25,15 +27,29 @@ from scrapy.settings import Settings
 logger = logging.getLogger(__name__)
 
 
+class InvalidNegotiatedProtocol(ProtocolError):
+
+    def __init__(self, negotiated_protocol: str) -> None:
+        self.negotiated_protocol = negotiated_protocol
+
+    def __str__(self) -> str:
+        return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}'
+
+
+@implementer(IHandshakeListener)
 class H2ClientProtocol(Protocol):
-    def __init__(self, uri: URI, settings: Settings) -> None:
+    def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None:
         """
         Arguments:
             uri -- URI of the base url to which HTTP/2 Connection will be made.
                 uri is used to verify that incoming client requests have correct
                 base URL.
             settings -- Scrapy project settings
+            conn_lost_deferred -- Deferred fires with the reason: Failure to notify
+                that connection was lost
         """
+        self._conn_lost_deferred = conn_lost_deferred
+
         config = H2Configuration(client_side=True, header_encoding='utf-8')
         self.conn = H2Connection(config=config)
 
@@ -55,9 +71,9 @@ class H2ClientProtocol(Protocol):
         # We use simple FIFO policy to handle pending requests
         self._active_streams = 0
 
-        # Save an instance of ProtocolError raised by hyper-h2
-        # We pass this instance to the streams ResponseFailed() failure
-        self._protocol_error: Optional[ProtocolError] = None
+        # Save an instance of errors raised which lead to losing the connection
+        # We pass these instances to the streams ResponseFailed() failure
+        self._conn_lost_errors: List[BaseException] = []
 
         self.metadata: H2ConnectionMetadataDict = {
             'certificate': None,
@@ -136,6 +152,12 @@ class H2ClientProtocol(Protocol):
 
         # Add the stream to the request pool
         self._pending_request_stream_pool.append(stream)
+
+        # If we are connection and receive a request
+        # There is a good chance that the connection was IDLE
+        # Hence, we need to initiate pending requests
+        if self.is_connected:
+            self._send_pending_requests()
         return d
 
     def connectionMade(self) -> None:
@@ -149,6 +171,19 @@ class H2ClientProtocol(Protocol):
         self.conn.initiate_connection()
         self._write_to_transport()
 
+    def _lose_connection_with_error(self, errors: List[BaseException]):
+        """Helper function to lose the connection with the error sent as a
+        reason"""
+        self._conn_lost_errors += errors
+        self.transport.loseConnection()
+
+    def handshakeCompleted(self):
+        """We close the connection with InvalidNegotiatedProtocol exception
+        when the connection was not made via h2 protocol"""
+        negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8')
+        if negotiated_protocol != 'h2':
+            self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)])
+
     def dataReceived(self, data: bytes) -> None:
         try:
             events = self.conn.receive_data(data)
@@ -157,10 +192,7 @@ class H2ClientProtocol(Protocol):
             # Save this error as ultimately the connection will be dropped
             # internally by hyper-h2. Saved error will be passed to all the streams
             # closed with the connection.
-            self._protocol_error = e
-
-            # We lose the transport connection here
-            self.transport.loseConnection()
+            self._lose_connection_with_error([e])
         finally:
             self._write_to_transport()
 
@@ -168,17 +200,17 @@ class H2ClientProtocol(Protocol):
         """Called by Twisted when the transport connection is lost.
         No need to write anything to transport here.
         """
-        errors = []
+        # Notify the connection pool instance such that no new requests are
+        # sent over current connection
         if not reason.check(connectionDone):
-            logger.warning("Connection lost with reason " + str(reason))
-            errors.append(reason)
+            self._conn_lost_errors.append(reason)
 
-        if self._protocol_error:
-            errors.append(self._protocol_error)
+        if self._conn_lost_deferred:
+            self._conn_lost_deferred.callback(self._conn_lost_errors)
 
         for stream in self.streams.values():
             if stream.request_sent:
-                stream.close(StreamCloseReason.CONNECTION_LOST, errors, from_protocol=True)
+                stream.close(StreamCloseReason.CONNECTION_LOST, self._conn_lost_errors, from_protocol=True)
             else:
                 stream.close(StreamCloseReason.INACTIVE, from_protocol=True)
 
@@ -242,10 +274,15 @@ class H2ClientProtocol(Protocol):
                 stream.receive_window_update()
 
 
+@implementer(IProtocolNegotiationFactory)
 class H2ClientFactory(Factory):
-    def __init__(self, uri: URI, settings: Settings) -> None:
+    def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None:
         self.uri = uri
         self.settings = settings
+        self.conn_lost_deferred = conn_lost_deferred
 
     def buildProtocol(self, addr) -> H2ClientProtocol:
-        return H2ClientProtocol(self.uri, self.settings)
+        return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred)
+
+    def acceptableProtocols(self) -> List[bytes]:
+        return [b'h2']
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index a2f0e2aa4..16d54b2fc 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -44,7 +44,7 @@ class InvalidHostname(Exception):
 
 
 class StreamCloseReason(Enum):
-    # Received a StreamEnded event
+    # Received a StreamEnded event from the remote
     ENDED = 1
 
     # Received a StreamReset event -- ended abruptly
@@ -329,7 +329,7 @@ class Stream:
     def close(
         self,
         reason: StreamCloseReason,
-        errors: Optional[List[Exception]] = None,
+        errors: Optional[List[BaseException]] = None,
         from_protocol: bool = False
     ) -> None:
         """Based on the reason sent we will handle each case.
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 05f4889ef..7fcea58c5 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -9,7 +9,7 @@ from urllib.parse import urlencode
 
 from h2.exceptions import InvalidBodyLengthError
 from twisted.internet import reactor
-from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError
+from twisted.internet.defer import CancelledError, DeferredList, inlineCallbacks
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
 from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate
 from twisted.python.failure import Failure

From 1dd27a92fa53be87c71df43bd6f3043a225a2c10 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Tue, 14 Jul 2020 17:58:22 +0530
Subject: [PATCH 078/716] feat: Idle Timeout for H2Connection (240s)

---
 scrapy/core/http2/protocol.py       | 60 ++++++++++++++++++++++++-----
 scrapy/core/http2/stream.py         |  3 +-
 tests/test_http2_client_protocol.py | 10 +++--
 3 files changed, 60 insertions(+), 13 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 5d859d475..bf41a2805 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -6,15 +6,18 @@ from typing import Dict, List, Optional
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
+from h2.errors import ErrorCodes
 from h2.events import (
     Event, DataReceived, ResponseReceived, SettingsAcknowledged,
     StreamEnded, StreamReset, WindowUpdated
 )
 from h2.exceptions import ProtocolError
 from twisted.internet.defer import Deferred
+from twisted.internet.error import TimeoutError
 from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
 from twisted.internet.protocol import connectionDone, Factory, Protocol
 from twisted.internet.ssl import Certificate
+from twisted.protocols.policies import TimeoutMixin
 from twisted.python.failure import Failure
 from twisted.web.client import URI
 from zope.interface import implementer
@@ -37,7 +40,9 @@ class InvalidNegotiatedProtocol(ProtocolError):
 
 
 @implementer(IHandshakeListener)
-class H2ClientProtocol(Protocol):
+class H2ClientProtocol(Protocol, TimeoutMixin):
+    IDLE_TIMEOUT = 240
+
     def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None:
         """
         Arguments:
@@ -71,6 +76,10 @@ class H2ClientProtocol(Protocol):
         # We use simple FIFO policy to handle pending requests
         self._active_streams = 0
 
+        # Flag to keep track if settings were acknowledged by the remote
+        # This ensures that we have established a HTTP/2 connection
+        self._settings_acknowledged = False
+
         # Save an instance of errors raised which lead to losing the connection
         # We pass these instances to the streams ResponseFailed() failure
         self._conn_lost_errors: List[BaseException] = []
@@ -84,12 +93,12 @@ class H2ClientProtocol(Protocol):
         }
 
     @property
-    def is_connected(self) -> bool:
+    def h2_connected(self) -> bool:
         """Boolean to keep track of the connection status.
         This is used while initiating pending streams to make sure
         that we initiate stream only during active HTTP/2 Connection
         """
-        return bool(self.transport.connected)
+        return bool(self.transport.connected) and self._settings_acknowledged
 
     @property
     def allowed_max_concurrent_streams(self) -> int:
@@ -111,7 +120,7 @@ class H2ClientProtocol(Protocol):
         while (
             self._pending_request_stream_pool
             and self._active_streams < self.allowed_max_concurrent_streams
-            and self.is_connected
+            and self.h2_connected
         ):
             self._active_streams += 1
             stream = self._pending_request_stream_pool.popleft()
@@ -140,6 +149,9 @@ class H2ClientProtocol(Protocol):
         """ Write data to the underlying transport connection
         from the HTTP2 connection instance if any
         """
+        # Reset the idle timeout as connection is still actively sending data
+        self.resetTimeout()
+
         data = self.conn.data_to_send()
         self.transport.write(data)
 
@@ -153,21 +165,23 @@ class H2ClientProtocol(Protocol):
         # Add the stream to the request pool
         self._pending_request_stream_pool.append(stream)
 
-        # If we are connection and receive a request
-        # There is a good chance that the connection was IDLE
-        # Hence, we need to initiate pending requests
-        if self.is_connected:
-            self._send_pending_requests()
+        # If we receive a request when connection is idle
+        # We need to initiate pending requests
+        self._send_pending_requests()
         return d
 
     def connectionMade(self) -> None:
         """Called by Twisted when the connection is established. We can start
         sending some data now: we should open with the connection preamble.
         """
+        # Initialize the timeout
+        self.setTimeout(self.IDLE_TIMEOUT)
+
         destination = self.transport.getPeer()
         logger.debug('Connection made to {}'.format(destination))
         self.metadata['ip_address'] = ipaddress.ip_address(destination.host)
 
+        # Initiate H2 Connection
         self.conn.initiate_connection()
         self._write_to_transport()
 
@@ -185,6 +199,9 @@ class H2ClientProtocol(Protocol):
             self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)])
 
     def dataReceived(self, data: bytes) -> None:
+        # Reset the idle timeout as connection is still actively receiving data
+        self.resetTimeout()
+
         try:
             events = self.conn.receive_data(data)
             self._handle_events(events)
@@ -196,10 +213,33 @@ class H2ClientProtocol(Protocol):
         finally:
             self._write_to_transport()
 
+    def timeoutConnection(self):
+        """Called when the connection times out.
+        We lose the connection with TimeoutError"""
+
+        # Check whether there are open streams. If there are, we're going to
+        # want to use the error code PROTOCOL_ERROR. If there aren't, use
+        # NO_ERROR.
+        if (
+            self.conn.open_outbound_streams > 0
+            or self.conn.open_inbound_streams > 0
+            or self._active_streams > 0
+        ):
+            error_code = ErrorCodes.PROTOCOL_ERROR
+        else:
+            error_code = ErrorCodes.NO_ERROR
+        self.conn.close_connection(error_code=error_code)
+        self._write_to_transport()
+
+        self._lose_connection_with_error([TimeoutError("Hello")])
+
     def connectionLost(self, reason: Failure = connectionDone) -> None:
         """Called by Twisted when the transport connection is lost.
         No need to write anything to transport here.
         """
+        # Cancel the timeout if not done yet
+        self.setTimeout(None)
+
         # Notify the connection pool instance such that no new requests are
         # sent over current connection
         if not reason.check(connectionDone):
@@ -250,6 +290,8 @@ class H2ClientProtocol(Protocol):
         self.streams[event.stream_id].receive_headers(event.headers)
 
     def settings_acknowledged(self, event: SettingsAcknowledged) -> None:
+        self._settings_acknowledged = True
+
         # Send off all the pending requests as now we have
         # established a proper HTTP/2 connection
         self._send_pending_requests()
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 16d54b2fc..f60691945 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -134,7 +134,8 @@ class Stream:
 
         def _cancel(_):
             # Close this stream as gracefully as possible
-            # Check if the stream has started
+            # If the associated request is initiated we reset this stream
+            # else we directly call close() method
             if self.request_sent:
                 self.reset_stream(StreamCloseReason.CANCELLED)
             else:
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 7fcea58c5..2833801e7 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -9,7 +9,7 @@ from urllib.parse import urlencode
 
 from h2.exceptions import InvalidBodyLengthError
 from twisted.internet import reactor
-from twisted.internet.defer import CancelledError, DeferredList, inlineCallbacks
+from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
 from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate
 from twisted.python.failure import Failure
@@ -174,6 +174,7 @@ class Https2ClientProtocolTestCase(TestCase):
         # Start server for testing
         self.hostname = u'localhost'
         context_factory = ssl_context_factory(self.key_file, self.certificate_file)
+
         server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname)
         self.server = yield server_endpoint.listen(self.site)
         self.port_number = self.server.getHost().port
@@ -186,17 +187,20 @@ class Https2ClientProtocolTestCase(TestCase):
             acceptableProtocols=[b'h2']
         )
         uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8'))
-        h2_client_factory = H2ClientFactory(uri, Settings())
+
+        self.conn_closed_deferred = Deferred()
+        h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)
         client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
         self.client = yield client_endpoint.connect(h2_client_factory)
 
     @inlineCallbacks
     def tearDown(self):
-        if self.client.is_connected:
+        if self.client.connected:
             yield self.client.transport.loseConnection()
             yield self.client.transport.abortConnection()
         yield self.server.stopListening()
         shutil.rmtree(self.temp_directory)
+        self.conn_closed_deferred = None
 
     def get_url(self, path):
         """

From e662762e6ab2f53ff2e31e21f34c078590f65aa9 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 15 Jul 2020 03:45:32 +0530
Subject: [PATCH 079/716] chore: Handle ConnectionTerminated event

---
 scrapy/core/http2/protocol.py | 44 +++++++++++++++++++++++++++--------
 scrapy/core/http2/stream.py   |  4 ++--
 2 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index bf41a2805..041908116 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -2,16 +2,18 @@ import ipaddress
 import itertools
 import logging
 from collections import deque
-from typing import Dict, List, Optional
+from ipaddress import IPv4Address, IPv6Address
+from typing import Dict, List, Optional, Union
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
 from h2.errors import ErrorCodes
 from h2.events import (
-    Event, DataReceived, ResponseReceived, SettingsAcknowledged,
-    StreamEnded, StreamReset, WindowUpdated
+    Event, ConnectionTerminated, DataReceived, ResponseReceived,
+    SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived,
+    WindowUpdated
 )
-from h2.exceptions import ProtocolError
+from h2.exceptions import H2Error, ProtocolError
 from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
 from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
@@ -30,7 +32,7 @@ from scrapy.settings import Settings
 logger = logging.getLogger(__name__)
 
 
-class InvalidNegotiatedProtocol(ProtocolError):
+class InvalidNegotiatedProtocol(H2Error):
 
     def __init__(self, negotiated_protocol: str) -> None:
         self.negotiated_protocol = negotiated_protocol
@@ -39,6 +41,15 @@ class InvalidNegotiatedProtocol(ProtocolError):
         return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}'
 
 
+class RemoteTerminatedConnection(H2Error):
+    def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], event: ConnectionTerminated):
+        self.remote_ip_address = remote_ip_address
+        self.terminate_event = event
+
+    def __str__(self) -> str:
+        return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address}'
+
+
 @implementer(IHandshakeListener)
 class H2ClientProtocol(Protocol, TimeoutMixin):
     IDLE_TIMEOUT = 240
@@ -194,8 +205,12 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
     def handshakeCompleted(self):
         """We close the connection with InvalidNegotiatedProtocol exception
         when the connection was not made via h2 protocol"""
-        negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8')
+        negotiated_protocol = self.transport.negotiatedProtocol
+        if type(negotiated_protocol) is bytes:
+            negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8')
         if negotiated_protocol != 'h2':
+            # Here we have not initiated the connection yet
+            # So, no need to send a GOAWAY frame to the remote
             self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)])
 
     def dataReceived(self, data: bytes) -> None:
@@ -231,7 +246,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self.conn.close_connection(error_code=error_code)
         self._write_to_transport()
 
-        self._lose_connection_with_error([TimeoutError("Hello")])
+        self._lose_connection_with_error([
+            TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")
+        ])
 
     def connectionLost(self, reason: Failure = connectionDone) -> None:
         """Called by Twisted when the transport connection is lost.
@@ -267,7 +284,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             events -- A list of events that the remote peer triggered by sending data
         """
         for event in events:
-            if isinstance(event, DataReceived):
+            if isinstance(event, ConnectionTerminated):
+                self.connection_terminated(event)
+            elif isinstance(event, DataReceived):
                 self.data_received(event)
             elif isinstance(event, ResponseReceived):
                 self.response_received(event)
@@ -279,10 +298,15 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
                 self.window_updated(event)
             elif isinstance(event, SettingsAcknowledged):
                 self.settings_acknowledged(event)
-            else:
-                logger.debug('Received unhandled event {}'.format(event))
+            elif isinstance(event, UnknownFrameReceived):
+                logger.debug(f'UnknownFrameReceived: frame={event.frame}')
 
     # Event handler functions starts here
+    def connection_terminated(self, event: ConnectionTerminated) -> None:
+        self._lose_connection_with_error([
+            RemoteTerminatedConnection(self.metadata['ip_address'], event)
+        ])
+
     def data_received(self, event: DataReceived) -> None:
         self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length)
 
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index f60691945..15f081cab 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -5,7 +5,7 @@ from typing import List, Optional, Tuple, TYPE_CHECKING
 from urllib.parse import urlparse
 
 from h2.errors import ErrorCodes
-from h2.exceptions import StreamClosedError
+from h2.exceptions import H2Error, StreamClosedError
 from hpack import HeaderTuple
 from twisted.internet.defer import Deferred, CancelledError
 from twisted.internet.error import ConnectionClosed
@@ -32,7 +32,7 @@ class InactiveStreamClosed(ConnectionClosed):
         self.request = request
 
 
-class InvalidHostname(Exception):
+class InvalidHostname(H2Error):
 
     def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None:
         self.request = request

From 316620b517207b1082dd9c5b4ebfc7fbe745e3bb Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 22 Jul 2020 13:53:46 +0530
Subject: [PATCH 080/716] chore: pass spider as argument for request method

- download_maxsize and download_warnsize can now be extracted from the
spider directly and passed to the stream
- remove `partial` flag from the response as per RFC 7540 - Section
8.1.2.6
---
 scrapy/core/http2/protocol.py       | 12 +++++---
 scrapy/core/http2/stream.py         | 35 ++++++++++------------
 tests/test_http2_client_protocol.py | 46 +++++++++++++++++++----------
 3 files changed, 55 insertions(+), 38 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 041908116..feb034a0c 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -28,6 +28,7 @@ from scrapy.core.http2.stream import Stream, StreamCloseReason
 from scrapy.core.http2.types import H2ConnectionMetadataDict
 from scrapy.http import Request
 from scrapy.settings import Settings
+from scrapy.spiders import Spider
 
 logger = logging.getLogger(__name__)
 
@@ -71,7 +72,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
         # ID of the next request stream
         # Following the convention - 'Streams initiated by a client MUST
-        # use odd-numbered stream identifiers' (RFC 7540)
+        # use odd-numbered stream identifiers' (RFC 7540 - Section 5.1.1)
         self._stream_id_generator = itertools.count(start=1, step=2)
 
         # Streams are stored in a dictionary keyed off their stream IDs
@@ -136,6 +137,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             self._active_streams += 1
             stream = self._pending_request_stream_pool.popleft()
             stream.initiate_request()
+            self._write_to_transport()
 
     def pop_stream(self, stream_id: int) -> Stream:
         """Perform cleanup when a stream is closed
@@ -145,13 +147,15 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self._send_pending_requests()
         return stream
 
-    def _new_stream(self, request: Request) -> Stream:
+    def _new_stream(self, request: Request, spider: Spider) -> Stream:
         """Instantiates a new Stream object
         """
         stream = Stream(
             stream_id=next(self._stream_id_generator),
             request=request,
             protocol=self,
+            download_maxsize=getattr(spider, 'download_maxsize', self.metadata['default_download_maxsize']),
+            download_warnsize=getattr(spider, 'download_warnsize', self.metadata['default_download_warnsize']),
         )
         self.streams[stream.stream_id] = stream
         return stream
@@ -166,11 +170,11 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         data = self.conn.data_to_send()
         self.transport.write(data)
 
-    def request(self, request: Request) -> Deferred:
+    def request(self, request: Request, spider: Spider) -> Deferred:
         if not isinstance(request, Request):
             raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}')
 
-        stream = self._new_stream(request)
+        stream = self._new_stream(request, spider)
         d = stream.get_response()
 
         # Add the stream to the request pool
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 15f081cab..133196797 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -83,7 +83,9 @@ class Stream:
         self,
         stream_id: int,
         request: Request,
-        protocol: "H2ClientProtocol"
+        protocol: "H2ClientProtocol",
+        download_maxsize: int = 0,
+        download_warnsize: int = 0
     ) -> None:
         """
         Arguments:
@@ -95,14 +97,8 @@ class Stream:
         self._request: Request = request
         self._protocol: "H2ClientProtocol" = protocol
 
-        self._download_maxsize = self._request.meta.get(
-            'download_maxsize',
-            self._protocol.metadata['default_download_maxsize']
-        )
-        self._download_warnsize = self._request.meta.get(
-            'download_warnsize',
-            self._protocol.metadata['default_download_warnsize']
-        )
+        self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
+        self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
 
         self.request_start_time = None
 
@@ -346,26 +342,28 @@ class Stream:
 
         self.stream_closed_server = True
 
-        flags = None
-        if b'Content-Length' not in self._response['headers']:
-            # Missing Content-Length - {twisted.web.http.PotentialDataLoss}
-            flags = ['partial']
+        # We do not check for Content-Length or Transfer-Encoding in response headers
+        # and add `partial` flag as in HTTP/1.1 as 'A request or response that includes
+        # a payload body can include a content-length header field' (RFC 7540 - Section 8.1.2.6)
 
         # NOTE: Order of handling the events is important here
         # As we immediately cancel the request when maxsize is exceeded while
         # receiving DATA_FRAME's when we have received the headers (not
         # having Content-Length)
         if reason is StreamCloseReason.MAXSIZE_EXCEEDED:
-            expected_size = int(self._response['headers'].get(b'Content-Length', -1))
+            expected_size = int(self._response['headers'].get(
+                b'Content-Length',
+                self._response['flow_controlled_size'])
+            )
             error_msg = (
-                f'Cancelling download of {self._request.url}: expected response '
-                f'size ({expected_size}) larger than download max size ({self._download_maxsize}).'
+                f'Cancelling download of {self._request.url}: received response '
+                f'size ({expected_size}) larger than download max size ({self._download_maxsize})'
             )
             logger.error(error_msg)
             self._deferred_response.errback(CancelledError(error_msg))
 
         elif reason is StreamCloseReason.ENDED:
-            self._fire_response_deferred(flags)
+            self._fire_response_deferred()
 
         # Stream was abruptly ended here
         elif reason is StreamCloseReason.CANCELLED:
@@ -398,7 +396,7 @@ class Stream:
                 f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}'
             ))
 
-    def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None:
+    def _fire_response_deferred(self) -> None:
         """Builds response from the self._response dict
         and fires the response deferred callback with the
         generated response instance"""
@@ -416,7 +414,6 @@ class Stream:
             headers=self._response['headers'],
             body=body,
             request=self._request,
-            flags=flags,
             certificate=self._protocol.metadata['certificate'],
             ip_address=self._protocol.metadata['ip_address'],
         )
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 2833801e7..d0386f7f8 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -23,6 +23,7 @@ from scrapy.core.http2.protocol import H2ClientFactory
 from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
 from scrapy.http import Request, Response, JsonRequest
 from scrapy.settings import Settings
+from scrapy.spiders import Spider
 from tests.mockserver import ssl_context_factory, LeafResource, Status
 
 
@@ -41,6 +42,14 @@ def make_html_body(val):
     return bytes(response, 'utf-8')
 
 
+class DummySpider(Spider):
+    name = 'dummy'
+    start_urls = []
+
+    def parse(self, response):
+        print(response)
+
+
 class Data:
     SMALL_SIZE = 1024  # 1 KB
     LARGE_SIZE = 1024 ** 2  # 1 MB
@@ -210,6 +219,9 @@ class Https2ClientProtocolTestCase(TestCase):
         assert len(path) > 0 and (path[0] == '/' or path[0] == '&')
         return f'{self.scheme}://{self.hostname}:{self.port_number}{path}'
 
+    def make_request(self, request: Request) -> Deferred:
+        return self.client.request(request, DummySpider())
+
     @staticmethod
     def _check_repeat(get_deferred, count):
         d_list = []
@@ -233,7 +245,7 @@ class Https2ClientProtocolTestCase(TestCase):
             content_length = int(response.headers.get('Content-Length'))
             self.assertEqual(len(response.body), content_length)
 
-        d = self.client.request(request)
+        d = self.make_request(request)
         d.addCallback(check_response)
         d.addErrback(self.fail)
         return d
@@ -273,7 +285,7 @@ class Https2ClientProtocolTestCase(TestCase):
         expected_extra_data,
         expected_status: int
     ):
-        d = self.client.request(request)
+        d = self.make_request(request)
 
         def assert_response(response: Response):
             self.assertEqual(response.status, expected_status)
@@ -355,7 +367,7 @@ class Https2ClientProtocolTestCase(TestCase):
             self.assertEqual(response.status, 499)
             self.assertEqual(response.request, request)
 
-        d = self.client.request(request)
+        d = self.make_request(request)
         d.addCallback(assert_response)
         d.addErrback(self.fail)
         d.cancel()
@@ -367,8 +379,13 @@ class Https2ClientProtocolTestCase(TestCase):
 
         def assert_cancelled_error(failure):
             self.assertIsInstance(failure.value, CancelledError)
+            error_pattern = re.compile(
+                rf'Cancelling download of {request.url}: received response '
+                rf'size \(\d*\) larger than download max size \(1000\)'
+            )
+            self.assertEqual(len(re.findall(error_pattern, str(failure.value))), 1)
 
-        d = self.client.request(request)
+        d = self.make_request(request)
         d.addCallback(self.fail)
         d.addErrback(assert_cancelled_error)
         return d
@@ -385,7 +402,7 @@ class Https2ClientProtocolTestCase(TestCase):
                 for error in failure.value.reasons
             ))
 
-        d = self.client.request(request)
+        d = self.make_request(request)
         d.addCallback(self.fail)
         d.addErrback(assert_failure)
         return d
@@ -397,10 +414,9 @@ class Https2ClientProtocolTestCase(TestCase):
             self.assertEqual(response.status, 200)
             self.assertEqual(response.body, Data.NO_CONTENT_LENGTH)
             self.assertEqual(response.request, request)
-            self.assertIn('partial', response.flags)
             self.assertNotIn('Content-Length', response.headers)
 
-        d = self.client.request(request)
+        d = self.make_request(request)
         d.addCallback(assert_content_length)
         d.addErrback(self.fail)
         return d
@@ -413,7 +429,7 @@ class Https2ClientProtocolTestCase(TestCase):
         expected_body
     ):
         with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm:
-            response = yield self.client.request(request)
+            response = yield self.make_request(request)
             self.assertEqual(response.status, 200)
             self.assertEqual(response.request, request)
             self.assertEqual(response.body, expected_body)
@@ -469,13 +485,13 @@ class Https2ClientProtocolTestCase(TestCase):
 
         # Send 100 request (we do not check the result)
         for _ in range(100):
-            d = self.client.request(Request(self.get_url('/get-data-html-small')))
+            d = self.make_request(Request(self.get_url('/get-data-html-small')))
             d.addBoth(lambda _: None)
             d_list.append(d)
 
         # Now send 10 extra request and save the response deferred in a list
         for _ in range(10):
-            d = self.client.request(Request(self.get_url('/get-data-html-small')))
+            d = self.make_request(Request(self.get_url('/get-data-html-small')))
             d.addCallback(self.fail)
             d.addErrback(assert_inactive_stream)
             d_list.append(d)
@@ -488,7 +504,7 @@ class Https2ClientProtocolTestCase(TestCase):
 
     def test_invalid_request_type(self):
         with self.assertRaises(TypeError):
-            self.client.request('https://InvalidDataTypePassed.com')
+            self.make_request('https://InvalidDataTypePassed.com')
 
     def test_query_parameters(self):
         params = {
@@ -504,7 +520,7 @@ class Https2ClientProtocolTestCase(TestCase):
             data = json.loads(str(response.body, content_encoding))
             self.assertEqual(data, params)
 
-        d = self.client.request(request)
+        d = self.make_request(request)
         d.addCallback(assert_query_params)
         d.addErrback(self.fail)
 
@@ -517,7 +533,7 @@ class Https2ClientProtocolTestCase(TestCase):
         d_list = []
         for status in [200, 404]:
             request = Request(self.get_url(f'/status?n={status}'))
-            d = self.client.request(request)
+            d = self.make_request(request)
             d.addCallback(assert_response_status, status)
             d.addErrback(self.fail)
             d_list.append(d)
@@ -537,7 +553,7 @@ class Https2ClientProtocolTestCase(TestCase):
             self.assertIsInstance(response.ip_address, IPv4Address)
             self.assertEqual(str(response.ip_address), '127.0.0.1')
 
-        d = self.client.request(request)
+        d = self.make_request(request)
         d.addCallback(assert_metadata)
         d.addErrback(self.fail)
 
@@ -553,7 +569,7 @@ class Https2ClientProtocolTestCase(TestCase):
             self.assertIn('127.0.0.1', error_msg)
             self.assertIn(str(request), error_msg)
 
-        d = self.client.request(request)
+        d = self.make_request(request)
         d.addCallback(self.fail)
         d.addErrback(assert_invalid_hostname)
         return d

From 3685e99cca47a5006258cd4246f255216797641f Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 22 Jul 2020 14:47:20 +0530
Subject: [PATCH 081/716] test: http2 connection timeout

---
 tests/test_http2_client_protocol.py | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index d0386f7f8..746eef4d6 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -18,8 +18,9 @@ from twisted.web.client import URI
 from twisted.web.http import Request as TxRequest
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.static import File
+from twisted.internet.error import TimeoutError
 
-from scrapy.core.http2.protocol import H2ClientFactory
+from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
 from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
 from scrapy.http import Request, Response, JsonRequest
 from scrapy.settings import Settings
@@ -134,6 +135,11 @@ class NoContentLengthHeader(LeafResource):
         request.finish()
 
 
+class TimeoutResponse(LeafResource):
+    def render_GET(self, request: TxRequest):
+        return NOT_DONE_YET
+
+
 class QueryParams(LeafResource):
     def render_GET(self, request: TxRequest):
         request.setHeader('Content-Type', 'application/json; charset=UTF-8')
@@ -172,6 +178,7 @@ class Https2ClientProtocolTestCase(TestCase):
         r.putChild(b'no-content-length-header', NoContentLengthHeader())
         r.putChild(b'status', Status())
         r.putChild(b'query-params', QueryParams())
+        r.putChild(b'timeout', TimeoutResponse())
         return r
 
     @inlineCallbacks
@@ -590,3 +597,22 @@ class Https2ClientProtocolTestCase(TestCase):
         ]
 
         return DeferredList(d_list, fireOnOneErrback=True)
+
+    def test_connection_timeout(self):
+        request = Request(self.get_url('/timeout'))
+        d = self.make_request(request)
+
+        # Update the timer to 1s to test connection timeout
+        self.client.setTimeout(1)
+
+        def assert_timeout_error(failure: Failure):
+            for err in failure.value.reasons:
+                if isinstance(err, TimeoutError):
+                    self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err))
+                    break
+            else:
+                self.fail()
+
+        d.addCallback(self.fail)
+        d.addErrback(assert_timeout_error)
+        return d

From 9fffb801ed3dedbb3935c811c7d61ed953ff22dc Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 8 Jul 2020 20:18:38 +0530
Subject: [PATCH 082/716] feat: H2Agent, H2ConnectionPool base implementation

---
 scrapy/core/downloader/handlers/http2.py | 55 ++++++++++++++++++++
 scrapy/core/http2/agent.py               | 64 ++++++++++++++++++++++++
 2 files changed, 119 insertions(+)
 create mode 100644 scrapy/core/downloader/handlers/http2.py
 create mode 100644 scrapy/core/http2/agent.py

diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
new file mode 100644
index 000000000..0dc06f4d8
--- /dev/null
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -0,0 +1,55 @@
+import warnings
+
+from scrapy.core.downloader.tls import openssl_methods
+from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
+from scrapy.http.request import Request
+from scrapy.settings import Settings
+from scrapy.utils.misc import create_instance, load_object
+
+
+class H2DownloadHandler:
+    def __init__(self, settings: Settings, crawler=None):
+        self._crawler = crawler
+
+        from twisted.internet import reactor
+        self._pool = H2ConnectionPool(reactor, settings)
+
+        self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
+        self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+        # try method-aware context factory
+        try:
+            self._context_factory = create_instance(
+                objcls=self._context_factory_cls,
+                settings=settings,
+                crawler=crawler,
+                method=self._ssl_method,
+            )
+        except TypeError:
+            # use context factory defaults
+            self._context_factory = create_instance(
+                objcls=self._context_factory_cls,
+                settings=settings,
+                crawler=crawler,
+            )
+            msg = """
+         '%s' does not accept `method` argument (type OpenSSL.SSL method,\
+         e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
+         Please upgrade your context factory class to handle them or ignore them.""" % (
+                settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
+            warnings.warn(msg)
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler.settings, crawler)
+
+    def download_request(self, request: Request, spider):
+        from twisted.internet import reactor
+
+        agent = H2Agent(reactor, self._pool, self._context_factory)
+        d = agent.request(request)
+
+        def print_result(result):
+            print(result)
+
+        d.addCallback(print_result)
+        return d
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
new file mode 100644
index 000000000..e14e5d633
--- /dev/null
+++ b/scrapy/core/http2/agent.py
@@ -0,0 +1,64 @@
+from typing import Dict, Tuple
+
+from twisted.internet import defer
+from twisted.internet.base import ReactorBase
+from twisted.internet.defer import Deferred
+from twisted.internet.endpoints import SSL4ClientEndpoint, optionsForClientTLS
+from twisted.web.client import URI, BrowserLikePolicyForHTTPS
+
+from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
+from scrapy.http.request import Request
+from scrapy.settings import Settings
+from scrapy.utils.python import to_bytes, to_unicode
+
+
+class H2ConnectionPool:
+    def __init__(self, reactor: ReactorBase, settings: Settings) -> None:
+        self._reactor = reactor
+        self.settings = settings
+        self._connections: Dict[Tuple, H2ClientProtocol] = {}
+
+    def get_connection(self, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred:
+        key = (uri.scheme, uri.host, uri.port)
+        conn = self._connections.get(key, None)
+        if conn:
+            return defer.succeed(conn)
+        return self._new_connection(key, uri, endpoint)
+
+    def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred:
+        factory = H2ClientFactory(uri, self.settings)
+        d = endpoint.connect(factory)
+
+        def put_connection(conn: H2ClientProtocol) -> H2ClientProtocol:
+            self._connections[key] = conn
+            return conn
+
+        d.addCallback(put_connection)
+        return d
+
+    def _remove_connection(self, key) -> None:
+        conn = self._connections.pop(key)
+        conn.loseConnection()
+
+
+class H2Agent:
+    def __init__(
+        self, reactor: ReactorBase, pool: H2ConnectionPool,
+        context_factory=BrowserLikePolicyForHTTPS()
+    ) -> None:
+        self._reactor = reactor
+        self._pool = pool
+        self._context_factory = context_factory
+
+    def request(self, request: Request) -> Deferred:
+        uri = URI.fromBytes(to_bytes(request.url, encoding='ascii'))
+        # options = optionsForClientTLS(hostname=to_unicode(uri.host), acceptableProtocols=[b'h2'])
+        # Hacky fix: Use options instead of self._context_factory to make endpoint work for HTTP/2
+        endpoint = SSL4ClientEndpoint(self._reactor, to_unicode(uri.host), uri.port, self._context_factory)
+        d = self._pool.get_connection(uri, endpoint)
+
+        def cb_connected(conn: H2ClientProtocol):
+            return conn.request(request)
+
+        d.addCallback(cb_connected)
+        return d

From 8252a6f8d8930ce23bdf8a6f51038b4ce49d1968 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 15 Jul 2020 04:58:59 +0530
Subject: [PATCH 083/716] fix: H2Agent not able to connect via SSL

- add H2WrappedContextFactory class which wraps the context factory
passed to H2Agent and updates the SSL context acceptable protocols list
to only h2
---
 scrapy/core/downloader/handlers/http2.py |  1 +
 scrapy/core/http2/agent.py               | 66 +++++++++++++++---------
 2 files changed, 42 insertions(+), 25 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 0dc06f4d8..c8b401e80 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -50,6 +50,7 @@ class H2DownloadHandler:
 
         def print_result(result):
             print(result)
+            return result
 
         d.addCallback(print_result)
         return d
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index e14e5d633..566c074c2 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -1,15 +1,19 @@
-from typing import Dict, Tuple
+from typing import Dict, Tuple, Optional
 
 from twisted.internet import defer
+from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions
 from twisted.internet.base import ReactorBase
 from twisted.internet.defer import Deferred
-from twisted.internet.endpoints import SSL4ClientEndpoint, optionsForClientTLS
-from twisted.web.client import URI, BrowserLikePolicyForHTTPS
+from twisted.internet.endpoints import SSL4ClientEndpoint
+from twisted.python.failure import Failure
+from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
+from twisted.web.iweb import IPolicyForHTTPS
+from zope.interface import implementer
+from zope.interface.verify import verifyObject
 
 from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
 from scrapy.http.request import Request
 from scrapy.settings import Settings
-from scrapy.utils.python import to_bytes, to_unicode
 
 
 class H2ConnectionPool:
@@ -26,39 +30,51 @@ class H2ConnectionPool:
         return self._new_connection(key, uri, endpoint)
 
     def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred:
-        factory = H2ClientFactory(uri, self.settings)
+        conn_lost_deferred = Deferred()
+        conn_lost_deferred.addCallback(self._remove_connection, key)
+
+        factory = H2ClientFactory(uri, self.settings, conn_lost_deferred)
         d = endpoint.connect(factory)
-
-        def put_connection(conn: H2ClientProtocol) -> H2ClientProtocol:
-            self._connections[key] = conn
-            return conn
-
-        d.addCallback(put_connection)
+        d.addCallback(self.put_connection, key)
         return d
 
-    def _remove_connection(self, key) -> None:
-        conn = self._connections.pop(key)
-        conn.loseConnection()
+    def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol:
+        self._connections[key] = conn
+        return conn
+
+    def _remove_connection(self, reason: Failure, key: Tuple) -> None:
+        self._connections.pop(key)
+
+
+@implementer(IPolicyForHTTPS)
+class H2WrappedContextFactory:
+    def __init__(self, context_factory) -> None:
+        verifyObject(IPolicyForHTTPS, context_factory)
+        self._wrapped_context_factory = context_factory
+
+    def creatorForNetloc(self, hostname, port) -> ClientTLSOptions:
+        options = self._wrapped_context_factory.creatorForNetloc(hostname, port)
+        _setAcceptableProtocols(options._ctx, [b'h2'])
+        return options
 
 
 class H2Agent:
     def __init__(
         self, reactor: ReactorBase, pool: H2ConnectionPool,
-        context_factory=BrowserLikePolicyForHTTPS()
+        context_factory=BrowserLikePolicyForHTTPS(),
+        connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
     ) -> None:
         self._reactor = reactor
         self._pool = pool
-        self._context_factory = context_factory
+        self._context_factory = H2WrappedContextFactory(context_factory)
+        self._endpoint_factory = _StandardEndpointFactory(
+            self._reactor, self._context_factory,
+            connect_timeout, bind_address
+        )
 
     def request(self, request: Request) -> Deferred:
-        uri = URI.fromBytes(to_bytes(request.url, encoding='ascii'))
-        # options = optionsForClientTLS(hostname=to_unicode(uri.host), acceptableProtocols=[b'h2'])
-        # Hacky fix: Use options instead of self._context_factory to make endpoint work for HTTP/2
-        endpoint = SSL4ClientEndpoint(self._reactor, to_unicode(uri.host), uri.port, self._context_factory)
+        uri = URI.fromBytes(bytes(request.url, encoding='utf-8'))
+        endpoint = self._endpoint_factory.endpointForURI(uri)
         d = self._pool.get_connection(uri, endpoint)
-
-        def cb_connected(conn: H2ClientProtocol):
-            return conn.request(request)
-
-        d.addCallback(cb_connected)
+        d.addCallback(lambda conn: conn.request(request))
         return d

From 62ce842afc8ef829ffd6f164712a8a9413eb9e1d Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 15 Jul 2020 07:50:53 +0530
Subject: [PATCH 084/716] fix: multiple h2 connections to same uri

- When multiple requests are sent to H2ConnectionPool to the same uri
while the connection is in connecting state -- multiple connections were
establised.
- Fixed the bug using a deque of all the request deferred's which fire
with the H2ClientProtocol (connection) instance when connection is
established
---
 scrapy/core/http2/agent.py | 55 ++++++++++++++++++++++++++++++++------
 1 file changed, 47 insertions(+), 8 deletions(-)

diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index 566c074c2..7a8847c38 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -1,11 +1,11 @@
-from typing import Dict, Tuple, Optional
+from collections import deque
+from typing import Deque, Dict, List, Tuple, Optional
 
 from twisted.internet import defer
 from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions
 from twisted.internet.base import ReactorBase
 from twisted.internet.defer import Deferred
-from twisted.internet.endpoints import SSL4ClientEndpoint
-from twisted.python.failure import Failure
+from twisted.internet.endpoints import HostnameEndpoint
 from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
 from twisted.web.iweb import IPolicyForHTTPS
 from zope.interface import implementer
@@ -20,31 +20,70 @@ class H2ConnectionPool:
     def __init__(self, reactor: ReactorBase, settings: Settings) -> None:
         self._reactor = reactor
         self.settings = settings
+
+        # Store a dictionary which is used to get the respective
+        # H2ClientProtocolInstance using the  key as Tuple(scheme, hostname, port)
         self._connections: Dict[Tuple, H2ClientProtocol] = {}
 
-    def get_connection(self, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred:
+        # Save all requests that arrive before the connection is established
+        self._pending_requests: Dict[Tuple, Deque[Deferred]] = {}
+
+    def get_connection(self, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
         key = (uri.scheme, uri.host, uri.port)
+        if key in self._pending_requests:
+            # Received a request while connecting to remote
+            # Create a deferred which will fire with the H2ClientProtocol
+            # instance
+            d = Deferred()
+            self._pending_requests[key].append(d)
+            return d
+
+        # Check if we already have a connection to the remote
         conn = self._connections.get(key, None)
         if conn:
+            # Return this connection instance wrapped inside a deferred
             return defer.succeed(conn)
+
+        # No connection is established for the given URI
         return self._new_connection(key, uri, endpoint)
 
-    def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred:
+    def _new_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
+        self._pending_requests[key] = deque()
+
         conn_lost_deferred = Deferred()
         conn_lost_deferred.addCallback(self._remove_connection, key)
 
         factory = H2ClientFactory(uri, self.settings, conn_lost_deferred)
-        d = endpoint.connect(factory)
-        d.addCallback(self.put_connection, key)
+        conn_d = endpoint.connect(factory)
+        conn_d.addCallback(self.put_connection, key)
+
+        d = Deferred()
+        self._pending_requests[key].append(d)
         return d
 
     def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol:
         self._connections[key] = conn
+
+        # Now as we have established a proper HTTP/2 connection
+        # we fire all the deferred's with the connection instance
+        pending_requests = self._pending_requests.pop(key)
+        while pending_requests:
+            d = pending_requests.popleft()
+            d.callback(conn)
+
+        del pending_requests
+
         return conn
 
-    def _remove_connection(self, reason: Failure, key: Tuple) -> None:
+    def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None:
         self._connections.pop(key)
 
+        # Call the errback of all the pending requests for this connection
+        pending_requests = self._pending_requests.pop(key, None)
+        while pending_requests:
+            d = pending_requests.popleft()
+            d.errback(errors)
+
 
 @implementer(IPolicyForHTTPS)
 class H2WrappedContextFactory:

From 031bfc9c3bed6c4dbfa4b7fcd48a8fc15f3582fb Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 22 Jul 2020 15:01:59 +0530
Subject: [PATCH 085/716] feat(wip): ScrapyH2Agent, ScrapyProxyH2Agent

---
 scrapy/core/downloader/contextfactory.py  |  32 +++++
 scrapy/core/downloader/handlers/http11.py |  27 +---
 scrapy/core/downloader/handlers/http2.py  | 161 +++++++++++++++++-----
 scrapy/core/http2/agent.py                |  35 ++++-
 4 files changed, 190 insertions(+), 65 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 452242d47..c0463cfc7 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,8 +1,12 @@
 from OpenSSL import SSL
+import warnings
+
 from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
 from twisted.web.client import BrowserLikePolicyForHTTPS
 from twisted.web.iweb import IPolicyForHTTPS
 from zope.interface.declarations import implementer
+from scrapy.core.downloader.tls import openssl_methods
+from scrapy.utils.misc import create_instance, load_object
 
 from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
 
@@ -92,3 +96,31 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
             trustRoot=platformTrust(),
             extraCertificateOptions={'method': self._ssl_method},
         )
+
+
+def load_context_factory_from_settings(settings, crawler):
+    ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
+    context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+    # try method-aware context factory
+    try:
+        context_factory = create_instance(
+            objcls=context_factory_cls,
+            settings=settings,
+            crawler=crawler,
+            method=ssl_method,
+        )
+    except TypeError:
+        # use context factory defaults
+        context_factory = create_instance(
+            objcls=context_factory_cls,
+            settings=settings,
+            crawler=crawler,
+        )
+        msg = """
+            '%s' does not accept `method` argument (type OpenSSL.SSL method,\
+            e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
+            Please upgrade your context factory class to handle them or ignore them.""" % (
+            settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
+        warnings.warn(msg)
+
+    return context_factory
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 22c9ac520..dac97ad29 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
 from zope.interface import implementer
 
 from scrapy import signals
-from scrapy.core.downloader.tls import openssl_methods
+from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
 from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
 from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
-from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import to_bytes, to_unicode
 
 
@@ -43,29 +42,7 @@ class HTTP11DownloadHandler:
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self._pool._factory.noisy = False
 
-        self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
-        self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
-        # try method-aware context factory
-        try:
-            self._contextFactory = create_instance(
-                objcls=self._contextFactoryClass,
-                settings=settings,
-                crawler=crawler,
-                method=self._sslMethod,
-            )
-        except TypeError:
-            # use context factory defaults
-            self._contextFactory = create_instance(
-                objcls=self._contextFactoryClass,
-                settings=settings,
-                crawler=crawler,
-            )
-            msg = """
- '%s' does not accept `method` argument (type OpenSSL.SSL method,\
- e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
- Please upgrade your context factory class to handle them or ignore them.""" % (
-                settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
-            warnings.warn(msg)
+        self._contextFactory = load_context_factory_from_settings(settings, crawler)
         self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
         self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
         self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index c8b401e80..81ea78e69 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -1,10 +1,19 @@
 import warnings
+from time import time
+from typing import Optional, Tuple
+from urllib.parse import urldefrag
 
-from scrapy.core.downloader.tls import openssl_methods
+from twisted.internet.base import ReactorBase
+from twisted.internet.error import TimeoutError
+from twisted.web.client import URI
+
+from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
+from scrapy.core.downloader.webclient import _parse
 from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
-from scrapy.http.request import Request
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.http import Request, Response
 from scrapy.settings import Settings
-from scrapy.utils.misc import create_instance, load_object
+from scrapy.spiders import Spider
 
 
 class H2DownloadHandler:
@@ -13,44 +22,128 @@ class H2DownloadHandler:
 
         from twisted.internet import reactor
         self._pool = H2ConnectionPool(reactor, settings)
-
-        self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
-        self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
-        # try method-aware context factory
-        try:
-            self._context_factory = create_instance(
-                objcls=self._context_factory_cls,
-                settings=settings,
-                crawler=crawler,
-                method=self._ssl_method,
-            )
-        except TypeError:
-            # use context factory defaults
-            self._context_factory = create_instance(
-                objcls=self._context_factory_cls,
-                settings=settings,
-                crawler=crawler,
-            )
-            msg = """
-         '%s' does not accept `method` argument (type OpenSSL.SSL method,\
-         e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
-         Please upgrade your context factory class to handle them or ignore them.""" % (
-                settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
-            warnings.warn(msg)
+        self._context_factory = load_context_factory_from_settings(settings, crawler)
+        self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
+        self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
+        self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
 
     @classmethod
     def from_crawler(cls, crawler):
         return cls(crawler.settings, crawler)
 
-    def download_request(self, request: Request, spider):
+    def download_request(self, request: Request, spider: Spider):
+        agent = ScrapyH2Agent(
+            context_factory=self._context_factory,
+            pool=self._pool,
+            maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
+            warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
+            crawler=self._crawler
+        )
+        return agent.download_request(request, spider)
+
+    def close(self) -> None:
+        self._pool.close_connections()
+
+
+class ScrapyProxyH2Agent(H2Agent):
+    def __init__(
+        self, reactor: ReactorBase,
+        proxy_uri: URI, pool: H2ConnectionPool,
+        connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
+    ) -> None:
+        super(ScrapyProxyH2Agent, self).__init__(
+            reactor=reactor,
+            pool=pool,
+            connect_timeout=connect_timeout,
+            bind_address=bind_address
+        )
+        self._proxy_uri = proxy_uri
+
+    @staticmethod
+    def get_key(uri: URI) -> Tuple:
+        return "http-proxy", uri.host, uri.port
+
+
+class ScrapyH2Agent:
+    _Agent = H2Agent
+    _ProxyAgent = ScrapyProxyH2Agent
+
+    def __init__(
+        self, context_factory,
+        connect_timeout=10,
+        bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None,
+        maxsize: int = 0, warnsize: int = 0,
+        crawler=None
+    ) -> None:
+        self._context_factory = context_factory
+        self._connect_timeout = connect_timeout
+        self._bind_address = bind_address
+        self._pool = pool
+        self._maxsize = maxsize
+        self._warnsize = warnsize
+        self._crawler = crawler
+
+    def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:
         from twisted.internet import reactor
+        bind_address = request.meta.get('bindaddress') or self._bind_address
+        proxy = request.meta.get('proxy')
+        if proxy:
+            _, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
+            scheme = _parse(request.url)[0]
+            proxy_host = str(proxy_host, 'utf-8')
+            omit_connect_timeout = b'noconnect' in proxy_params
+            if omit_connect_timeout:
+                warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
+                              "If you use Crawlera, it doesn't require this mode anymore, "
+                              "so you should update scrapy-crawlera to 1.3.0+ "
+                              "and remove '?noconnect' from the Crawlera URL.",
+                              ScrapyDeprecationWarning)
 
-        agent = H2Agent(reactor, self._pool, self._context_factory)
-        d = agent.request(request)
+            if scheme == b'https' and not omit_connect_timeout:
+                proxy_auth = request.headers.get(b'Proxy-Authorization', None)
+                proxy_conf = (proxy_host, proxy_port, proxy_auth)
 
-        def print_result(result):
-            print(result)
-            return result
+                # TODO: Return TunnelingAgent instance
+            else:
+                return self._ProxyAgent(
+                    reactor=reactor,
+                    proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')),
+                    connect_timeout=timeout,
+                    bind_address=bind_address,
+                    pool=self._pool
+                )
 
-        d.addCallback(print_result)
+        return self._Agent(
+            reactor=reactor,
+            context_factory=self._context_factory,
+            connect_timeout=timeout,
+            bind_address=bind_address,
+            pool=self._pool
+        )
+
+    def download_request(self, request: Request, spider: Spider):
+        from twisted.internet import reactor
+        timeout = request.meta.get('download_timeout') or self._connect_timeout
+        agent = self._get_agent(request, timeout)
+
+        start_time = time()
+        d = agent.request(request, spider)
+        d.addCallback(self._cb_latency, request, start_time)
+
+        timeout_cl = reactor.callLater(timeout, d.cancel)
+        d.addBoth(self._cb_timeout, request, timeout, timeout_cl)
         return d
+
+    @staticmethod
+    def _cb_latency(response: Response, request: Request, start_time: float):
+        request.meta['download_latency'] = time() - start_time
+        return response
+
+    @staticmethod
+    def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl):
+        if timeout_cl.active():
+            timeout_cl.cancel()
+            return response
+
+        url = urldefrag(request.url)[0]
+        raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index 7a8847c38..c7a49fd42 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -6,7 +6,9 @@ from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOption
 from twisted.internet.base import ReactorBase
 from twisted.internet.defer import Deferred
 from twisted.internet.endpoints import HostnameEndpoint
+from twisted.python.failure import Failure
 from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
+from twisted.web.error import SchemeNotSupported
 from twisted.web.iweb import IPolicyForHTTPS
 from zope.interface import implementer
 from zope.interface.verify import verifyObject
@@ -14,6 +16,7 @@ from zope.interface.verify import verifyObject
 from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
 from scrapy.http.request import Request
 from scrapy.settings import Settings
+from scrapy.spiders import Spider
 
 
 class H2ConnectionPool:
@@ -28,8 +31,7 @@ class H2ConnectionPool:
         # Save all requests that arrive before the connection is established
         self._pending_requests: Dict[Tuple, Deque[Deferred]] = {}
 
-    def get_connection(self, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
-        key = (uri.scheme, uri.host, uri.port)
+    def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
         if key in self._pending_requests:
             # Received a request while connecting to remote
             # Create a deferred which will fire with the H2ClientProtocol
@@ -84,6 +86,15 @@ class H2ConnectionPool:
             d = pending_requests.popleft()
             d.errback(errors)
 
+    def close_connections(self) -> None:
+        """Close all the HTTP/2 connections and remove them from pool
+
+        Returns:
+            Deferred that fires when all connections have been closed
+        """
+        for conn in self._connections.values():
+            conn.transport.loseConnection()
+
 
 @implementer(IPolicyForHTTPS)
 class H2WrappedContextFactory:
@@ -111,9 +122,21 @@ class H2Agent:
             connect_timeout, bind_address
         )
 
-    def request(self, request: Request) -> Deferred:
+    def _get_endpoint(self, uri: URI):
+        return self._endpoint_factory.endpointForURI(uri)
+
+    @staticmethod
+    def get_key(uri: URI) -> Tuple:
+        return uri.scheme, uri.host, uri.port
+
+    def request(self, request: Request, spider: Spider) -> Deferred:
         uri = URI.fromBytes(bytes(request.url, encoding='utf-8'))
-        endpoint = self._endpoint_factory.endpointForURI(uri)
-        d = self._pool.get_connection(uri, endpoint)
-        d.addCallback(lambda conn: conn.request(request))
+        try:
+            endpoint = self._get_endpoint(uri)
+        except SchemeNotSupported:
+            return defer.fail(Failure())
+
+        key = self.get_key(uri)
+        d = self._pool.get_connection(key, uri, endpoint)
+        d.addCallback(lambda conn: conn.request(request, spider))
         return d

From 92bec38591fccb523c2e643aef70a1f6cd7267ea Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 29 Jul 2020 13:43:59 +0530
Subject: [PATCH 086/716] feat: MethodNotAllowed405, Content-Length header

- add tests to check for Content-Length header
- raise MethodNotAllowed405 when remote send 'HTTP/2.0 405 Method Not
Allowed'
---
 scrapy/core/downloader/handlers/http2.py | 15 +++-----
 scrapy/core/http2/agent.py               |  2 +-
 scrapy/core/http2/protocol.py            | 32 +++++++++++++----
 scrapy/core/http2/stream.py              | 14 ++++++--
 tests/test_http2_client_protocol.py      | 46 ++++++++++++++++++++++--
 5 files changed, 85 insertions(+), 24 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 81ea78e69..e9cc5ebbc 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -3,6 +3,7 @@ from time import time
 from typing import Optional, Tuple
 from urllib.parse import urldefrag
 
+from twisted.internet.defer import Deferred
 from twisted.internet.base import ReactorBase
 from twisted.internet.error import TimeoutError
 from twisted.web.client import URI
@@ -23,9 +24,6 @@ class H2DownloadHandler:
         from twisted.internet import reactor
         self._pool = H2ConnectionPool(reactor, settings)
         self._context_factory = load_context_factory_from_settings(settings, crawler)
-        self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
-        self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
-        self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -35,8 +33,6 @@ class H2DownloadHandler:
         agent = ScrapyH2Agent(
             context_factory=self._context_factory,
             pool=self._pool,
-            maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
-            warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
             crawler=self._crawler
         )
         return agent.download_request(request, spider)
@@ -72,15 +68,12 @@ class ScrapyH2Agent:
         self, context_factory,
         connect_timeout=10,
         bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None,
-        maxsize: int = 0, warnsize: int = 0,
         crawler=None
     ) -> None:
         self._context_factory = context_factory
         self._connect_timeout = connect_timeout
         self._bind_address = bind_address
         self._pool = pool
-        self._maxsize = maxsize
-        self._warnsize = warnsize
         self._crawler = crawler
 
     def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:
@@ -121,7 +114,7 @@ class ScrapyH2Agent:
             pool=self._pool
         )
 
-    def download_request(self, request: Request, spider: Spider):
+    def download_request(self, request: Request, spider: Spider) -> Deferred:
         from twisted.internet import reactor
         timeout = request.meta.get('download_timeout') or self._connect_timeout
         agent = self._get_agent(request, timeout)
@@ -135,12 +128,12 @@ class ScrapyH2Agent:
         return d
 
     @staticmethod
-    def _cb_latency(response: Response, request: Request, start_time: float):
+    def _cb_latency(response: Response, request: Request, start_time: float) -> Response:
         request.meta['download_latency'] = time() - start_time
         return response
 
     @staticmethod
-    def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl):
+    def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl) -> Response:
         if timeout_cl.active():
             timeout_cl.cancel()
             return response
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index c7a49fd42..e62eef263 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -93,7 +93,7 @@ class H2ConnectionPool:
             Deferred that fires when all connections have been closed
         """
         for conn in self._connections.values():
-            conn.transport.loseConnection()
+            conn.transport.abortConnection()
 
 
 @implementer(IPolicyForHTTPS)
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index feb034a0c..1ce8b6548 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -13,7 +13,7 @@ from h2.events import (
     SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived,
     WindowUpdated
 )
-from h2.exceptions import H2Error, ProtocolError
+from h2.exceptions import H2Error
 from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
 from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
@@ -39,7 +39,7 @@ class InvalidNegotiatedProtocol(H2Error):
         self.negotiated_protocol = negotiated_protocol
 
     def __str__(self) -> str:
-        return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}'
+        return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}'
 
 
 class RemoteTerminatedConnection(H2Error):
@@ -48,7 +48,15 @@ class RemoteTerminatedConnection(H2Error):
         self.terminate_event = event
 
     def __str__(self) -> str:
-        return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address}'
+        return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address!r}'
+
+
+class MethodNotAllowed405(H2Error):
+    def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]):
+        self.remote_ip_address = remote_ip_address
+
+    def __str__(self) -> str:
+        return f"MethodNotAllowed405: Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}"
 
 
 @implementer(IHandshakeListener)
@@ -217,14 +225,25 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             # So, no need to send a GOAWAY frame to the remote
             self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)])
 
+    def _check_received_data(self, data: bytes) -> None:
+        """Checks for edge cases where the connection to remote fails
+        without raising an appropriate H2Error
+
+        Arguments:
+            data -- Data received from the remote
+        """
+        if data.startswith(b'HTTP/2.0 405 Method Not Allowed'):
+            raise MethodNotAllowed405(self.metadata['ip_address'])
+
     def dataReceived(self, data: bytes) -> None:
         # Reset the idle timeout as connection is still actively receiving data
         self.resetTimeout()
 
         try:
+            self._check_received_data(data)
             events = self.conn.receive_data(data)
             self._handle_events(events)
-        except ProtocolError as e:
+        except H2Error as e:
             # Save this error as ultimately the connection will be dropped
             # internally by hyper-h2. Saved error will be passed to all the streams
             # closed with the connection.
@@ -271,9 +290,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
         for stream in self.streams.values():
             if stream.request_sent:
-                stream.close(StreamCloseReason.CONNECTION_LOST, self._conn_lost_errors, from_protocol=True)
+                close_reason = StreamCloseReason.CONNECTION_LOST
             else:
-                stream.close(StreamCloseReason.INACTIVE, from_protocol=True)
+                close_reason = StreamCloseReason.INACTIVE
+            stream.close(close_reason, self._conn_lost_errors, from_protocol=True)
 
         self._active_streams -= len(self.streams)
         self.streams.clear()
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 133196797..5bffa67e7 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -189,12 +189,15 @@ class Stream:
         headers = [
             (':method', self._request.method),
             (':authority', url.netloc),
-            (':scheme', 'https'),
+            (':scheme', self._protocol.metadata['uri'].scheme),
             (':path', path),
         ]
 
         for name, value in self._request.headers.items():
-            headers.append((name, value[0]))
+            headers.append((str(name, 'utf-8'), str(value[0], 'utf-8')))
+
+        if b'Content-Length' not in self._request.headers.keys():
+            headers.append(('Content-Length', str(len(self._request.body))))
 
         return headers
 
@@ -337,6 +340,10 @@ class Stream:
         if not isinstance(reason, StreamCloseReason):
             raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}')
 
+        # Have default value of errors as an empty list as
+        # some cases can add a list of exceptions
+        errors = errors or []
+
         if not from_protocol:
             self._protocol.pop_stream(self.stream_id)
 
@@ -387,7 +394,8 @@ class Stream:
             self._deferred_response.errback(ResponseFailed(errors))
 
         elif reason is StreamCloseReason.INACTIVE:
-            self._deferred_response.errback(InactiveStreamClosed(self._request))
+            errors.insert(0, InactiveStreamClosed(self._request))
+            self._deferred_response.errback(ResponseFailed(errors))
 
         elif reason is StreamCloseReason.INVALID_HOSTNAME:
             self._deferred_response.errback(InvalidHostname(
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 746eef4d6..4926ada14 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -11,14 +11,14 @@ from h2.exceptions import InvalidBodyLengthError
 from twisted.internet import reactor
 from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
+from twisted.internet.error import TimeoutError
 from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate
 from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
-from twisted.web.client import URI
+from twisted.web.client import ResponseFailed, URI
 from twisted.web.http import Request as TxRequest
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.static import File
-from twisted.internet.error import TimeoutError
 
 from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
 from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
@@ -152,6 +152,19 @@ class QueryParams(LeafResource):
         return bytes(json.dumps(query_params), 'utf-8')
 
 
+class RequestHeaders(LeafResource):
+    """Sends all the headers received as a response"""
+
+    def render_GET(self, request: TxRequest):
+        request.setHeader('Content-Type', 'application/json; charset=UTF-8')
+        request.setHeader('Content-Encoding', 'UTF-8')
+        headers = {}
+        for k, v in request.requestHeaders.getAllRawHeaders():
+            headers[str(k, 'utf-8')] = str(v[0], 'utf-8')
+
+        return bytes(json.dumps(headers), 'utf-8')
+
+
 def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
     with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate:
         pem = ''.join(key.readlines()) + ''.join(certificate.readlines())
@@ -179,6 +192,7 @@ class Https2ClientProtocolTestCase(TestCase):
         r.putChild(b'status', Status())
         r.putChild(b'query-params', QueryParams())
         r.putChild(b'timeout', TimeoutResponse())
+        r.putChild(b'request-headers', RequestHeaders())
         return r
 
     @inlineCallbacks
@@ -488,7 +502,11 @@ class Https2ClientProtocolTestCase(TestCase):
         d_list = []
 
         def assert_inactive_stream(failure):
-            self.assertIsNotNone(failure.check(InactiveStreamClosed))
+            self.assertIsNotNone(failure.check(ResponseFailed))
+            self.assertTrue(any(
+                isinstance(e, InactiveStreamClosed)
+                for e in failure.value.reasons
+            ))
 
         # Send 100 request (we do not check the result)
         for _ in range(100):
@@ -616,3 +634,25 @@ class Https2ClientProtocolTestCase(TestCase):
         d.addCallback(self.fail)
         d.addErrback(assert_timeout_error)
         return d
+
+    def test_request_headers_received(self):
+        request = Request(self.get_url('/request-headers'), headers={
+            'header-1': 'header value 1',
+            'header-2': 'header value 2'
+        })
+        d = self.make_request(request)
+
+        def assert_request_headers(response: Response):
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.request, request)
+
+            response_headers = json.loads(str(response.body, 'utf-8'))
+            self.assertIsInstance(response_headers, dict)
+            for k, v in request.headers.items():
+                k, v = str(k, 'utf-8'), str(v[0], 'utf-8')
+                self.assertIn(k, response_headers)
+                self.assertEqual(v, response_headers[k])
+
+        d.addErrback(self.fail)
+        d.addCallback(assert_request_headers)
+        return d

From e8342996f6273b6b60ffc26a67e90dec34e96dfd Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 29 Jul 2020 13:51:01 +0530
Subject: [PATCH 087/716] test: H2DownloadHandler

Following tests are skipped as Content-Length header not matching the
data received is considered as a ProtocolError
- test_download_broken_content_cause_data_loss
- test_download_broken_chunked_content_cause_data_loss
- test_download_broken_content_allow_data_loss
- test_download_broken_chunked_content_allow_data_loss
- test_download_broken_content_allow_data_loss_via_setting
- test_download_broken_chunked_content_allow_data_loss_via_setting

BREAKING CHANGES
The following tests currently fail
- test_content_length_zero_bodyless_post_request_headers
- test_host_header_seted_in_request_headers
- test_download_with_maxsize_very_large_file
---
 tests/test_downloader_handlers.py | 108 +++++++++++++++++++++++++++---
 1 file changed, 99 insertions(+), 9 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 51deb20f4..f6add82dc 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -23,8 +23,8 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
+from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
-
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Headers, Request
 from scrapy.http.response.text import TextResponse
@@ -33,7 +33,6 @@ from scrapy.spiders import Spider
 from scrapy.utils.misc import create_instance
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler, skip_if_no_boto
-
 from tests.mockserver import MockServer, ssl_context_factory, Echo
 from tests.spiders import SingleRequestSpider
 
@@ -131,6 +130,7 @@ class ContentLengthHeaderResource(resource.Resource):
     A testing resource which renders itself as the value of the Content-Length
     header from the request.
     """
+
     def render(self, request):
         return request.requestHeaders.getRawHeaders(b"content-length")[0]
 
@@ -142,6 +142,7 @@ class ChunkedResource(resource.Resource):
             request.write(b"chunked ")
             request.write(b"content\n")
             request.finish()
+
         reactor.callLater(0, response)
         return server.NOT_DONE_YET
 
@@ -155,6 +156,7 @@ class BrokenChunkedResource(resource.Resource):
             # Disable terminating chunk on finish.
             request.chunked = False
             closeConnection(request)
+
         reactor.callLater(0, response)
         return server.NOT_DONE_YET
 
@@ -186,6 +188,7 @@ class EmptyContentTypeHeaderResource(resource.Resource):
     A testing resource which renders itself as the value of request body
     without content-type header in response.
     """
+
     def render(self, request):
         request.setHeader("content-type", "")
         return request.content.read()
@@ -197,12 +200,12 @@ class LargeChunkedFileResource(resource.Resource):
             for i in range(1024):
                 request.write(b"x" * 1024)
             request.finish()
+
         reactor.callLater(0, response)
         return server.NOT_DONE_YET
 
 
 class HttpTestCase(unittest.TestCase):
-
     scheme = 'http'
     download_handler_cls = HTTPDownloadHandler
 
@@ -230,10 +233,12 @@ class HttpTestCase(unittest.TestCase):
         r.putChild(b"echo", Echo())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
-        self.host = 'localhost'
+        self.host = u'localhost'
         if self.scheme == 'https':
+            # Using WrappingFactory do not enable HTTP/2 failing all the
+            # tests with H2DownloadHandler
             self.port = reactor.listenSSL(
-                0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile),
+                0, self.site, ssl_context_factory(self.keyfile, self.certfile),
                 interface=self.host)
         else:
             self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
@@ -330,6 +335,7 @@ class HttpTestCase(unittest.TestCase):
         https://github.com/kennethreitz/requests/issues/405
         https://bugs.python.org/issue14721
         """
+
         def _test(response):
             self.assertEqual(response.body, b'0')
 
@@ -514,6 +520,30 @@ class Https11TestCase(Http11TestCase):
             yield download_handler.close()
 
 
+class Https2TestCase(Https11TestCase):
+    scheme = 'https'
+    download_handler_cls = H2DownloadHandler
+    HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
+
+    def test_download_broken_content_cause_data_loss(self, url='broken'):
+        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+    def test_download_broken_chunked_content_cause_data_loss(self):
+        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+    def test_download_broken_content_allow_data_loss(self, url='broken'):
+        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+    def test_download_broken_chunked_content_allow_data_loss(self):
+        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+    def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
+        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+    def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
+        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+
 class Https11WrongHostnameTestCase(Http11TestCase):
     scheme = 'https'
 
@@ -526,6 +556,23 @@ class Https11WrongHostnameTestCase(Http11TestCase):
     certfile = 'keys/example-com.cert.pem'
 
 
+class Https2WrongHostnameTestCase(Https2TestCase):
+    tls_log_message = (
+        'SSL connection certificate: issuer "/C=XW/ST=XW/L=The '
+        'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", '
+        'subject "/C=XW/ST=XW/L=The '
+        'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"'
+    )
+
+    # above tests use a server certificate for "localhost",
+    # client connection to "localhost" too.
+    # here we test that even if the server certificate is for another domain,
+    # "www.example.com" in this case,
+    # the tests still pass
+    keyfile = 'keys/example-com.key.pem'
+    certfile = 'keys/example-com.cert.pem'
+
+
 class Https11InvalidDNSId(Https11TestCase):
     """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
 
@@ -534,6 +581,14 @@ class Https11InvalidDNSId(Https11TestCase):
         self.host = '127.0.0.1'
 
 
+class Https2InvalidDNSId(Https2TestCase):
+    """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
+
+    def setUp(self):
+        super(Https2InvalidDNSId, self).setUp()
+        self.host = '127.0.0.1'
+
+
 class Https11InvalidDNSPattern(Https11TestCase):
     """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
 
@@ -552,6 +607,24 @@ class Https11InvalidDNSPattern(Https11TestCase):
         super(Https11InvalidDNSPattern, self).setUp()
 
 
+class Https2InvalidDNSPattern(Https2TestCase):
+    """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
+
+    keyfile = 'keys/localhost.ip.key'
+    certfile = 'keys/localhost.ip.crt'
+
+    def setUp(self):
+        try:
+            from service_identity.exceptions import CertificateError  # noqa: F401
+        except ImportError:
+            raise unittest.SkipTest("cryptography lib is too old")
+        self.tls_log_message = (
+            'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
+            'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
+        )
+        super(Https2InvalidDNSPattern, self).setUp()
+
+
 class Https11CustomCiphers(unittest.TestCase):
     scheme = 'https'
     download_handler_cls = HTTP11DownloadHandler
@@ -565,10 +638,9 @@ class Https11CustomCiphers(unittest.TestCase):
         FilePath(self.tmpname).child("file").setContent(b"0123456789")
         r = static.File(self.tmpname)
         self.site = server.Site(r, timeout=None)
-        self.wrapper = WrappingFactory(self.site)
         self.host = 'localhost'
         self.port = reactor.listenSSL(
-            0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
+            0, self.site, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
             interface=self.host)
         self.portno = self.port.getHost().port
         crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
@@ -593,6 +665,11 @@ class Https11CustomCiphers(unittest.TestCase):
         return d
 
 
+class Https2CustomCiphers(Https11CustomCiphers):
+    scheme = 'https'
+    download_handler_cls = H2DownloadHandler
+
+
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
 
@@ -644,6 +721,10 @@ class Http11MockServerTestCase(unittest.TestCase):
         self.assertTrue(reason, 'finished')
 
 
+class Http2MockServerTestCase(Http11MockServerTestCase):
+    """HTTP 2.0 test case with MockServer"""
+
+
 class UriResource(resource.Resource):
     """Return the full uri that was requested"""
 
@@ -734,6 +815,11 @@ class Http11ProxyTestCase(HttpProxyTestCase):
         self.assertIn(domain, timeout.osError)
 
 
+# TODO:
+class Http2ProxyTestCase(Http11ProxyTestCase):
+    download_handler_cls = H2DownloadHandler
+
+
 class HttpDownloadHandlerMock:
 
     def __init__(self, *args, **kwargs):
@@ -931,7 +1017,6 @@ class S3TestCase(unittest.TestCase):
 
 
 class BaseFTPTestCase(unittest.TestCase):
-
     username = "scrapy"
     password = "passwd"
     req_meta = {"ftp_user": username, "ftp_password": password}
@@ -969,6 +1054,7 @@ class BaseFTPTestCase(unittest.TestCase):
         def _clean(data):
             self.download_handler.client.transport.loseConnection()
             return data
+
         deferred.addCallback(_clean)
         if callback:
             deferred.addCallback(callback)
@@ -985,6 +1071,7 @@ class BaseFTPTestCase(unittest.TestCase):
             self.assertEqual(r.status, 200)
             self.assertEqual(r.body, b'I have the power!')
             self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']})
+
         return self._add_test_callbacks(d, _test)
 
     def test_ftp_download_path_with_spaces(self):
@@ -998,6 +1085,7 @@ class BaseFTPTestCase(unittest.TestCase):
             self.assertEqual(r.status, 200)
             self.assertEqual(r.body, b'Moooooooooo power!')
             self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']})
+
         return self._add_test_callbacks(d, _test)
 
     def test_ftp_download_notexist(self):
@@ -1007,6 +1095,7 @@ class BaseFTPTestCase(unittest.TestCase):
 
         def _test(r):
             self.assertEqual(r.status, 404)
+
         return self._add_test_callbacks(d, _test)
 
     def test_ftp_local_filename(self):
@@ -1027,6 +1116,7 @@ class BaseFTPTestCase(unittest.TestCase):
             with open(local_fname, "rb") as f:
                 self.assertEqual(f.read(), b"I have the power!")
             os.remove(local_fname)
+
         return self._add_test_callbacks(d, _test)
 
 
@@ -1043,11 +1133,11 @@ class FTPTestCase(BaseFTPTestCase):
 
         def _test(r):
             self.assertEqual(r.type, ConnectionLost)
+
         return self._add_test_callbacks(d, errback=_test)
 
 
 class AnonymousFTPTestCase(BaseFTPTestCase):
-
     username = "anonymous"
     req_meta = {}
 

From 19f2b4b53dd51044083a9749f00366f41ed795c7 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 29 Jul 2020 17:25:59 +0530
Subject: [PATCH 088/716] refactor: AcceptableProtocolsContextFactory

- rename H2WrappedContextFactory to AcceptableProtocolsContextFactory
- AcceptableProtocolsContextFactory accepts an argument
acceptable_protocols which can be used to override the context factory
priority list of protocols during ALPN or NPN
---
 scrapy/core/downloader/handlers/http2.py | 2 +-
 scrapy/core/http2/agent.py               | 9 +++++----
 scrapy/core/http2/stream.py              | 2 +-
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index e9cc5ebbc..411e06a78 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -3,8 +3,8 @@ from time import time
 from typing import Optional, Tuple
 from urllib.parse import urldefrag
 
-from twisted.internet.defer import Deferred
 from twisted.internet.base import ReactorBase
+from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
 from twisted.web.client import URI
 
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index e62eef263..aa51508a5 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -97,14 +97,15 @@ class H2ConnectionPool:
 
 
 @implementer(IPolicyForHTTPS)
-class H2WrappedContextFactory:
-    def __init__(self, context_factory) -> None:
+class AcceptableProtocolsContextFactory:
+    def __init__(self, context_factory, acceptable_protocols: List[bytes]) -> None:
         verifyObject(IPolicyForHTTPS, context_factory)
         self._wrapped_context_factory = context_factory
+        self._acceptable_protocols = acceptable_protocols
 
     def creatorForNetloc(self, hostname, port) -> ClientTLSOptions:
         options = self._wrapped_context_factory.creatorForNetloc(hostname, port)
-        _setAcceptableProtocols(options._ctx, [b'h2'])
+        _setAcceptableProtocols(options._ctx, self._acceptable_protocols)
         return options
 
 
@@ -116,7 +117,7 @@ class H2Agent:
     ) -> None:
         self._reactor = reactor
         self._pool = pool
-        self._context_factory = H2WrappedContextFactory(context_factory)
+        self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2'])
         self._endpoint_factory = _StandardEndpointFactory(
             self._reactor, self._context_factory,
             connect_timeout, bind_address
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 5bffa67e7..acdd46320 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -85,7 +85,7 @@ class Stream:
         request: Request,
         protocol: "H2ClientProtocol",
         download_maxsize: int = 0,
-        download_warnsize: int = 0
+        download_warnsize: int = 0,
     ) -> None:
         """
         Arguments:

From a3fecaf07f9edd6a1bbac1ade825c23845c7e6b1 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Thu, 30 Jul 2020 15:45:27 +0530
Subject: [PATCH 089/716] test: fix host-name H2DownloadHandler tests

---
 tests/test_downloader_handlers.py | 33 +++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index f6add82dc..4dd32b6f5 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -312,16 +312,18 @@ class HttpTestCase(unittest.TestCase):
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
     def test_host_header_seted_in_request_headers(self):
-        def _test(response):
-            self.assertEqual(response.body, b'example.com')
-            self.assertEqual(request.headers.get('Host'), b'example.com')
+        host = self.host + ':' + str(self.portno)
 
-        request = Request(self.getURL('host'), headers={'Host': 'example.com'})
+        def _test(response):
+            self.assertEqual(response.body, bytes(host, 'utf-8'))
+            self.assertEqual(request.headers.get('Host'), bytes(host, 'utf-8'))
+
+        request = Request(self.getURL('host'), headers={'Host': host})
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEqual, b'example.com')
+        d.addCallback(self.assertEqual, b'localhost')
         return d
 
     def test_content_length_zero_bodyless_post_request_headers(self):
@@ -339,7 +341,7 @@ class HttpTestCase(unittest.TestCase):
         def _test(response):
             self.assertEqual(response.body, b'0')
 
-        request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'})
+        request = Request(self.getURL('contentlength'), method='POST')
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
     def test_content_length_zero_bodyless_post_only_one(self):
@@ -525,6 +527,25 @@ class Https2TestCase(Https11TestCase):
     download_handler_cls = H2DownloadHandler
     HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
 
+    @defer.inlineCallbacks
+    def test_download_with_maxsize_very_large_file(self):
+        with mock.patch('scrapy.core.http2.stream.logger') as logger:
+            request = Request(self.getURL('largechunkedfile'))
+
+            def check(logger):
+                logger.error.assert_called_once_with(mock.ANY)
+
+            d = self.download_request(request, Spider('foo', download_maxsize=1500))
+            yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+
+            # As the error message is logged in the dataReceived callback, we
+            # have to give a bit of time to the reactor to process the queue
+            # after closing the connection.
+            d = defer.Deferred()
+            d.addCallback(check)
+            reactor.callLater(.1, d.callback, logger)
+            yield d
+
     def test_download_broken_content_cause_data_loss(self, url='broken'):
         raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
 

From d707f8b5d94856ba13bd8e76acb1efb8de377f9c Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Thu, 30 Jul 2020 18:06:21 +0530
Subject: [PATCH 090/716] docs: mention H2DownloadHandler in settings.rst

---
 docs/topics/settings.rst | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 5178f272f..670b44f3c 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -620,6 +620,13 @@ handler (without replacement), place this in your ``settings.py``::
         'ftp': None,
     }
 
+The default https handler uses HTTP/1.x, to use HTTP/2.0 update :setting:`DOWNLOAD_HANDLERS`
+as::
+
+    DOWNLOAD_HANDLERS = {
+        'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
+    }
+
 .. setting:: DOWNLOAD_TIMEOUT
 
 DOWNLOAD_TIMEOUT
@@ -697,6 +704,14 @@ Optionally, this can be set per-request basis by using the
   If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``,
   the ``ResponseFailed([_DataLoss])`` failure will be retried as usual.
 
+.. warning::
+
+    This is ignored when :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`
+    is set as ``https`` download handler in :setting:`DOWNLOAD_HANDLERS`. In
+    case of data loss error the connection may be corrupted affecting other streams,
+    hence all streams return with the ``ResponseFailed([InvalidBodyLengthError])``
+    failure.
+
 .. setting:: DUPEFILTER_CLASS
 
 DUPEFILTER_CLASS

From 1cc8d5829fc1b1b10fd852db693ac44dc9be0ef1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 6 Aug 2020 13:52:47 +0200
Subject: [PATCH 091/716] Remove unneeded try-except

Exceptions only happen when find_spec gets a 2nd parameter.
---
 scrapy/commands/startproject.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 35b58090c..82ccda35e 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -42,10 +42,7 @@ class Command(ScrapyCommand):
 
     def _is_valid_name(self, project_name):
         def _module_exists(module_name):
-            try:
-                spec = find_spec(module_name)
-            except ModuleNotFoundError:
-                return False
+            spec = find_spec(module_name)
             return spec is not None and spec.loader is not None
 
         if not re.search(r'^[_a-zA-Z]\w*$', project_name):

From 13181ba7882f2aef3ea103a9c2391fd8f87fbb44 Mon Sep 17 00:00:00 2001
From: Jose Galdos <jose@bitmaker.la>
Date: Thu, 23 Jul 2020 18:45:45 -0500
Subject: [PATCH 092/716] Improve http status all on http error middleware

---
 scrapy/spidermiddlewares/httperror.py    |  2 +-
 tests/test_spidermiddleware_httperror.py | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 375042340..bf908d2f7 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -32,7 +32,7 @@ class HttpErrorMiddleware:
         if 200 <= response.status < 300:  # common case
             return
         meta = response.meta
-        if 'handle_httpstatus_all' in meta:
+        if meta.get('handle_httpstatus_all', False):
             return
         if 'handle_httpstatus_list' in meta:
             allowed_statuses = meta['handle_httpstatus_list']
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index e032b247c..f3e5478c4 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -139,6 +139,19 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
         self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
         self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
 
+    def test_httperror_allow_all_false(self):
+        crawler = get_crawler(_HttpErrorSpider)
+        mw = HttpErrorMiddleware.from_crawler(crawler)
+        request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False})
+        request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True})
+        res404 = self.res404.copy()
+        res404.request = request_httpstatus_false
+        res402 = self.res402.copy()
+        res402.request = request_httpstatus_true
+
+        self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider)
+        self.assertIsNone(mw.process_spider_input(res402, self.spider))
+
 
 class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
     def setUp(self):

From e0c3019d90f187482cd84b946b83c496411ec34b Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sun, 9 Aug 2020 16:19:35 +0530
Subject: [PATCH 093/716] fix: ScrapyProxyH2Agent

- add required test cases

BREAKING CHANGES
Presently the tests (in test_downloader_handlers.py)
1. test_download_without_proxy
2. test_download_with_proxy_https_timeout

collide with each other when run together. However, if both of the tests
are ran individually then both pass.
---
 scrapy/core/downloader/handlers/http2.py | 14 ++++++---
 scrapy/core/http2/agent.py               | 15 ++++++----
 scrapy/core/http2/stream.py              | 22 +++++++++++++--
 tests/test_downloader_handlers.py        | 36 ++++++++++++++++++++----
 4 files changed, 70 insertions(+), 17 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 411e06a78..d6e5cd1c1 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -6,7 +6,7 @@ from urllib.parse import urldefrag
 from twisted.internet.base import ReactorBase
 from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
-from twisted.web.client import URI
+from twisted.web.client import URI, BrowserLikePolicyForHTTPS
 
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
@@ -45,19 +45,24 @@ class ScrapyProxyH2Agent(H2Agent):
     def __init__(
         self, reactor: ReactorBase,
         proxy_uri: URI, pool: H2ConnectionPool,
+        context_factory=BrowserLikePolicyForHTTPS(),
         connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
     ) -> None:
         super(ScrapyProxyH2Agent, self).__init__(
             reactor=reactor,
             pool=pool,
+            context_factory=context_factory,
             connect_timeout=connect_timeout,
             bind_address=bind_address
         )
         self._proxy_uri = proxy_uri
 
-    @staticmethod
-    def get_key(uri: URI) -> Tuple:
-        return "http-proxy", uri.host, uri.port
+    def get_endpoint(self, uri: URI):
+        return self.endpoint_factory.endpointForURI(self._proxy_uri)
+
+    def get_key(self, uri: URI) -> Tuple:
+        """We use the proxy uri instead of uri obtained from request url"""
+        return "http-proxy", self._proxy_uri.host, self._proxy_uri.port
 
 
 class ScrapyH2Agent:
@@ -100,6 +105,7 @@ class ScrapyH2Agent:
             else:
                 return self._ProxyAgent(
                     reactor=reactor,
+                    context_factory=self._context_factory,
                     proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')),
                     connect_timeout=timeout,
                     bind_address=bind_address,
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index aa51508a5..f4ac29bc6 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -118,22 +118,25 @@ class H2Agent:
         self._reactor = reactor
         self._pool = pool
         self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2'])
-        self._endpoint_factory = _StandardEndpointFactory(
+        self.endpoint_factory = _StandardEndpointFactory(
             self._reactor, self._context_factory,
             connect_timeout, bind_address
         )
 
-    def _get_endpoint(self, uri: URI):
-        return self._endpoint_factory.endpointForURI(uri)
+    def get_endpoint(self, uri: URI):
+        return self.endpoint_factory.endpointForURI(uri)
 
-    @staticmethod
-    def get_key(uri: URI) -> Tuple:
+    def get_key(self, uri: URI) -> Tuple:
+        """
+        Arguments:
+            uri - URI obtained directly from request URL
+        """
         return uri.scheme, uri.host, uri.port
 
     def request(self, request: Request, spider: Spider) -> Deferred:
         uri = URI.fromBytes(bytes(request.url, encoding='utf-8'))
         try:
-            endpoint = self._get_endpoint(uri)
+            endpoint = self.get_endpoint(uri)
         except SchemeNotSupported:
             return defer.fail(Failure())
 
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index acdd46320..40ea07b63 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -185,14 +185,32 @@ class Stream:
         if url.query:
             path += '?' + url.query
 
+        # This pseudo-header field MUST NOT be empty for "http" or "https"
+        # URIs; "http" or "https" URIs that do not contain a path component
+        # MUST include a value of '/'. The exception to this rule is an
+        # OPTIONS request for an "http" or "https" URI that does not include
+        # a path component; these MUST include a ":path" pseudo-header field
+        # with a value of '*' (refer RFC 7540 - Section 8.1.2.3)
+        if not path:
+            if self._request.method == 'OPTIONS':
+                path = path or '*'
+            else:
+                path = path or '/'
+
         # Make sure pseudo-headers comes before all the other headers
         headers = [
             (':method', self._request.method),
             (':authority', url.netloc),
-            (':scheme', self._protocol.metadata['uri'].scheme),
-            (':path', path),
         ]
 
+        # The ":scheme" and ":path" pseudo-header fields MUST
+        # be omitted for CONNECT method (refer RFC 7540 - Section 8.3)
+        if self._request.method != 'CONNECT':
+            headers += [
+                (':scheme', self._protocol.metadata['uri'].scheme),
+                (':path', path),
+            ]
+
         for name, value in self._request.headers.items():
             headers.append((str(name, 'utf-8'), str(value[0], 'utf-8')))
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 4dd32b6f5..486614121 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -786,7 +786,10 @@ class HttpProxyTestCase(unittest.TestCase):
         def _test(response):
             self.assertEqual(response.status, 200)
             self.assertEqual(response.url, request.url)
-            self.assertEqual(response.body, b'http://example.com')
+            self.assertTrue(
+                response.body == b'http://example.com'  # HTTP/1.x
+                or response.body == b'/'  # HTTP/2
+            )
 
         http_proxy = self.getURL('')
         request = Request('http://example.com', meta={'proxy': http_proxy})
@@ -796,10 +799,13 @@ class HttpProxyTestCase(unittest.TestCase):
         def _test(response):
             self.assertEqual(response.status, 200)
             self.assertEqual(response.url, request.url)
-            self.assertEqual(response.body, b'https://example.com')
+            self.assertTrue(
+                response.body == b'http://example.com'  # HTTP/1.x
+                or response.body == b'/'  # HTTP/2
+            )
 
         http_proxy = '%s?noconnect' % self.getURL('')
-        request = Request('https://example.com', meta={'proxy': http_proxy})
+        request = Request('http://example.com', meta={'proxy': http_proxy})
         with self.assertWarnsRegex(ScrapyDeprecationWarning,
                                    r'Using HTTPS proxies in the noconnect mode is deprecated'):
             return self.download_request(request, Spider('foo')).addCallback(_test)
@@ -836,10 +842,30 @@ class Http11ProxyTestCase(HttpProxyTestCase):
         self.assertIn(domain, timeout.osError)
 
 
-# TODO:
-class Http2ProxyTestCase(Http11ProxyTestCase):
+class Https2ProxyTestCase(Http11ProxyTestCase):
+    # only used for HTTPS tests
+    keyfile = 'keys/localhost.key'
+    certfile = 'keys/localhost.crt'
+
+    scheme = 'https'
+    host = u'127.0.0.1'
+
     download_handler_cls = H2DownloadHandler
 
+    def setUp(self):
+        site = server.Site(UriResource(), timeout=None)
+        self.port = reactor.listenSSL(
+            0, site,
+            ssl_context_factory(self.keyfile, self.certfile),
+            interface=self.host
+        )
+        self.portno = self.port.getHost().port
+        self.download_handler = create_instance(self.download_handler_cls, None, get_crawler())
+        self.download_request = self.download_handler.download_request
+
+    def getURL(self, path):
+        return f"{self.scheme}://{self.host}:{self.portno}/{path}"
+
 
 class HttpDownloadHandlerMock:
 

From c67d6dea318d6f0915ae86d46ce367b6c1e8ee51 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Tue, 11 Aug 2020 04:39:41 +0530
Subject: [PATCH 094/716] fix: H2 docs, NotImplementedError for H2 Tunnel

---
 docs/topics/settings.rst                 | 15 ++++----
 scrapy/core/downloader/contextfactory.py |  7 ++--
 scrapy/core/downloader/handlers/http2.py | 44 +++++++++++-------------
 scrapy/core/http2/stream.py              |  5 +--
 tests/test_downloader_handlers.py        | 33 ++++++++++++------
 5 files changed, 54 insertions(+), 50 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 670b44f3c..bb543433c 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -620,8 +620,8 @@ handler (without replacement), place this in your ``settings.py``::
         'ftp': None,
     }
 
-The default https handler uses HTTP/1.x, to use HTTP/2.0 update :setting:`DOWNLOAD_HANDLERS`
-as::
+The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
+:setting:`DOWNLOAD_HANDLERS` as follows::
 
     DOWNLOAD_HANDLERS = {
         'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
@@ -706,11 +706,12 @@ Optionally, this can be set per-request basis by using the
 
 .. warning::
 
-    This is ignored when :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`
-    is set as ``https`` download handler in :setting:`DOWNLOAD_HANDLERS`. In
-    case of data loss error the connection may be corrupted affecting other streams,
-    hence all streams return with the ``ResponseFailed([InvalidBodyLengthError])``
-    failure.
+    This setting is ignored by the
+    :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`
+    download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss
+    error, the corresponding HTTP/2 connection may be corrupted, affecting other
+    requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])``
+    failure is always raised for every request that was using that connection.
 
 .. setting:: DUPEFILTER_CLASS
 
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index c0463cfc7..8dcba15ff 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,14 +1,13 @@
-from OpenSSL import SSL
 import warnings
 
+from OpenSSL import SSL
 from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
 from twisted.web.client import BrowserLikePolicyForHTTPS
 from twisted.web.iweb import IPolicyForHTTPS
 from zope.interface.declarations import implementer
-from scrapy.core.downloader.tls import openssl_methods
-from scrapy.utils.misc import create_instance, load_object
 
-from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
+from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions
+from scrapy.utils.misc import create_instance, load_object
 
 
 @implementer(IPolicyForHTTPS)
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index d6e5cd1c1..650af9778 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -6,15 +6,15 @@ from urllib.parse import urldefrag
 from twisted.internet.base import ReactorBase
 from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
-from twisted.web.client import URI, BrowserLikePolicyForHTTPS
+from twisted.web.client import BrowserLikePolicyForHTTPS, URI
 
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
 from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
+from scrapy.utils.python import to_bytes
 
 
 class H2DownloadHandler:
@@ -88,29 +88,25 @@ class ScrapyH2Agent:
         if proxy:
             _, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
             scheme = _parse(request.url)[0]
-            proxy_host = str(proxy_host, 'utf-8')
-            omit_connect_timeout = b'noconnect' in proxy_params
-            if omit_connect_timeout:
-                warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
-                              "If you use Crawlera, it doesn't require this mode anymore, "
-                              "so you should update scrapy-crawlera to 1.3.0+ "
-                              "and remove '?noconnect' from the Crawlera URL.",
-                              ScrapyDeprecationWarning)
+            proxy_host = proxy_host.decode()
+            omit_connect_tunnel = b'noconnect' in proxy_params
+            if omit_connect_tunnel:
+                warnings.warn("Using HTTPS proxies in the noconnect mode is not supported by the "
+                              "downloader handler. If you use Crawlera, it doesn't require this "
+                              "mode anymore, so you should update scrapy-crawlera to 1.3.0+ "
+                              "and remove '?noconnect' from the Crawlera URL.")
 
-            if scheme == b'https' and not omit_connect_timeout:
-                proxy_auth = request.headers.get(b'Proxy-Authorization', None)
-                proxy_conf = (proxy_host, proxy_port, proxy_auth)
-
-                # TODO: Return TunnelingAgent instance
-            else:
-                return self._ProxyAgent(
-                    reactor=reactor,
-                    context_factory=self._context_factory,
-                    proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')),
-                    connect_timeout=timeout,
-                    bind_address=bind_address,
-                    pool=self._pool
-                )
+            if scheme == b'https' and not omit_connect_tunnel:
+                # ToDo
+                raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported')
+            return self._ProxyAgent(
+                reactor=reactor,
+                context_factory=self._context_factory,
+                proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')),
+                connect_timeout=timeout,
+                bind_address=bind_address,
+                pool=self._pool
+            )
 
         return self._Agent(
             reactor=reactor,
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 40ea07b63..1e136fbd5 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -192,10 +192,7 @@ class Stream:
         # a path component; these MUST include a ":path" pseudo-header field
         # with a value of '*' (refer RFC 7540 - Section 8.1.2.3)
         if not path:
-            if self._request.method == 'OPTIONS':
-                path = path or '*'
-            else:
-                path = path or '/'
+            path = '*' if self._request.method == 'OPTIONS' else '/'
 
         # Make sure pseudo-headers comes before all the other headers
         headers = [
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 486614121..1cedf6b10 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -315,8 +315,8 @@ class HttpTestCase(unittest.TestCase):
         host = self.host + ':' + str(self.portno)
 
         def _test(response):
-            self.assertEqual(response.body, bytes(host, 'utf-8'))
-            self.assertEqual(request.headers.get('Host'), bytes(host, 'utf-8'))
+            self.assertEqual(response.body, to_bytes(host))
+            self.assertEqual(request.headers.get('Host'), to_bytes(host))
 
         request = Request(self.getURL('host'), headers={'Host': host})
         return self.download_request(request, Spider('foo')).addCallback(_test)
@@ -764,6 +764,7 @@ class UriResource(resource.Resource):
 
 class HttpProxyTestCase(unittest.TestCase):
     download_handler_cls = HTTPDownloadHandler
+    expected_http_proxy_request_body = b'http://example.com'
 
     def setUp(self):
         site = server.Site(UriResource(), timeout=None)
@@ -786,10 +787,7 @@ class HttpProxyTestCase(unittest.TestCase):
         def _test(response):
             self.assertEqual(response.status, 200)
             self.assertEqual(response.url, request.url)
-            self.assertTrue(
-                response.body == b'http://example.com'  # HTTP/1.x
-                or response.body == b'/'  # HTTP/2
-            )
+            self.assertEqual(response.body, self.expected_http_proxy_request_body)
 
         http_proxy = self.getURL('')
         request = Request('http://example.com', meta={'proxy': http_proxy})
@@ -799,13 +797,10 @@ class HttpProxyTestCase(unittest.TestCase):
         def _test(response):
             self.assertEqual(response.status, 200)
             self.assertEqual(response.url, request.url)
-            self.assertTrue(
-                response.body == b'http://example.com'  # HTTP/1.x
-                or response.body == b'/'  # HTTP/2
-            )
+            self.assertEqual(response.body, b'https://example.com')
 
         http_proxy = '%s?noconnect' % self.getURL('')
-        request = Request('http://example.com', meta={'proxy': http_proxy})
+        request = Request('https://example.com', meta={'proxy': http_proxy})
         with self.assertWarnsRegex(ScrapyDeprecationWarning,
                                    r'Using HTTPS proxies in the noconnect mode is deprecated'):
             return self.download_request(request, Spider('foo')).addCallback(_test)
@@ -851,6 +846,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
     host = u'127.0.0.1'
 
     download_handler_cls = H2DownloadHandler
+    expected_http_proxy_request_body = b'/'
 
     def setUp(self):
         site = server.Site(UriResource(), timeout=None)
@@ -866,6 +862,21 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
     def getURL(self, path):
         return f"{self.scheme}://{self.host}:{self.portno}/{path}"
 
+    def test_download_with_proxy_https_noconnect(self):
+        def _test(response):
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.url, request.url)
+            self.assertEqual(response.body, b'/')
+
+        http_proxy = '%s?noconnect' % self.getURL('')
+        request = Request('https://example.com', meta={'proxy': http_proxy})
+        with self.assertWarnsRegex(
+            Warning,
+            r'Using HTTPS proxies in the noconnect mode is not supported by the '
+            r'downloader handler.'
+        ):
+            return self.download_request(request, Spider('foo')).addCallback(_test)
+
 
 class HttpDownloadHandlerMock:
 

From 90f85a2b9b0cfac4a7a56a1926e9694ef7e2d299 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 11 Aug 2020 10:20:30 +0200
Subject: [PATCH 095/716] Enable Travis CI

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index b403ac54c..a3bd2e199 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,6 +3,7 @@ dist: xenial
 branches:
   only:
     - master
+    - http2  # Remove once merged into master
     - /^\d\.\d+$/
     - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:

From af73f141b23aabbef208a12fe95ae63c27105fda Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sun, 16 Aug 2020 11:26:10 +0530
Subject: [PATCH 096/716] refactor: move all http2 tests in separate files

---
 tests/test_download_handlers_http2.py | 158 ++++++++++++++++++++++++++
 tests/test_downloader_handlers.py     | 141 +----------------------
 2 files changed, 160 insertions(+), 139 deletions(-)
 create mode 100644 tests/test_download_handlers_http2.py

diff --git a/tests/test_download_handlers_http2.py b/tests/test_download_handlers_http2.py
new file mode 100644
index 000000000..583dc1d17
--- /dev/null
+++ b/tests/test_download_handlers_http2.py
@@ -0,0 +1,158 @@
+from unittest import mock
+
+from twisted.internet import defer, error, reactor
+from twisted.trial import unittest
+from twisted.web import server
+
+from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+from scrapy.http import Request
+from scrapy.spiders import Spider
+from scrapy.utils.misc import create_instance
+from scrapy.utils.test import get_crawler
+from tests.mockserver import ssl_context_factory
+from tests.test_downloader_handlers import (
+    Https11TestCase, Https11CustomCiphers,
+    Http11MockServerTestCase, Http11ProxyTestCase,
+    UriResource
+)
+
+
+class Https2TestCase(Https11TestCase):
+    scheme = 'https'
+    download_handler_cls = H2DownloadHandler
+    HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
+
+    @defer.inlineCallbacks
+    def test_download_with_maxsize_very_large_file(self):
+        with mock.patch('scrapy.core.http2.stream.logger') as logger:
+            request = Request(self.getURL('largechunkedfile'))
+
+            def check(logger):
+                logger.error.assert_called_once_with(mock.ANY)
+
+            d = self.download_request(request, Spider('foo', download_maxsize=1500))
+            yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
+
+            # As the error message is logged in the dataReceived callback, we
+            # have to give a bit of time to the reactor to process the queue
+            # after closing the connection.
+            d = defer.Deferred()
+            d.addCallback(check)
+            reactor.callLater(.1, d.callback, logger)
+            yield d
+
+    def test_download_broken_content_cause_data_loss(self, url='broken'):
+        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+    def test_download_broken_chunked_content_cause_data_loss(self):
+        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+    def test_download_broken_content_allow_data_loss(self, url='broken'):
+        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+    def test_download_broken_chunked_content_allow_data_loss(self):
+        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+    def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
+        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+    def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
+        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
+
+
+class Https2WrongHostnameTestCase(Https2TestCase):
+    tls_log_message = (
+        'SSL connection certificate: issuer "/C=XW/ST=XW/L=The '
+        'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", '
+        'subject "/C=XW/ST=XW/L=The '
+        'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"'
+    )
+
+    # above tests use a server certificate for "localhost",
+    # client connection to "localhost" too.
+    # here we test that even if the server certificate is for another domain,
+    # "www.example.com" in this case,
+    # the tests still pass
+    keyfile = 'keys/example-com.key.pem'
+    certfile = 'keys/example-com.cert.pem'
+
+
+class Https2InvalidDNSId(Https2TestCase):
+    """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
+
+    def setUp(self):
+        super(Https2InvalidDNSId, self).setUp()
+        self.host = '127.0.0.1'
+
+
+class Https2InvalidDNSPattern(Https2TestCase):
+    """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
+
+    keyfile = 'keys/localhost.ip.key'
+    certfile = 'keys/localhost.ip.crt'
+
+    def setUp(self):
+        try:
+            from service_identity.exceptions import CertificateError  # noqa: F401
+        except ImportError:
+            raise unittest.SkipTest("cryptography lib is too old")
+        self.tls_log_message = (
+            'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
+            'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
+        )
+        super(Https2InvalidDNSPattern, self).setUp()
+
+
+class Https2CustomCiphers(Https11CustomCiphers):
+    scheme = 'https'
+    download_handler_cls = H2DownloadHandler
+
+
+class Http2MockServerTestCase(Http11MockServerTestCase):
+    """HTTP 2.0 test case with MockServer"""
+
+
+class Https2ProxyTestCase(Http11ProxyTestCase):
+    # only used for HTTPS tests
+    keyfile = 'keys/localhost.key'
+    certfile = 'keys/localhost.crt'
+
+    scheme = 'https'
+    host = u'127.0.0.1'
+
+    download_handler_cls = H2DownloadHandler
+    expected_http_proxy_request_body = b'/'
+
+    def setUp(self):
+        site = server.Site(UriResource(), timeout=None)
+        self.port = reactor.listenSSL(
+            0, site,
+            ssl_context_factory(self.keyfile, self.certfile),
+            interface=self.host
+        )
+        self.portno = self.port.getHost().port
+        self.download_handler = create_instance(self.download_handler_cls, None, get_crawler())
+        self.download_request = self.download_handler.download_request
+
+    def getURL(self, path):
+        return f"{self.scheme}://{self.host}:{self.portno}/{path}"
+
+    def test_download_with_proxy_https_noconnect(self):
+        def _test(response):
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.url, request.url)
+            self.assertEqual(response.body, b'/')
+
+        http_proxy = '%s?noconnect' % self.getURL('')
+        request = Request('https://example.com', meta={'proxy': http_proxy})
+        with self.assertWarnsRegex(
+            Warning,
+            r'Using HTTPS proxies in the noconnect mode is not supported by the '
+            r'downloader handler.'
+        ):
+            return self.download_request(request, Spider('foo')).addCallback(_test)
+
+    @defer.inlineCallbacks
+    def test_download_with_proxy_https_timeout(self):
+        with self.assertRaises(NotImplementedError):
+            yield super(Https2ProxyTestCase, self).test_download_with_proxy_https_timeout()
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 1cedf6b10..8b2b2c32c 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -23,7 +23,6 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
-from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Headers, Request
@@ -315,8 +314,8 @@ class HttpTestCase(unittest.TestCase):
         host = self.host + ':' + str(self.portno)
 
         def _test(response):
-            self.assertEqual(response.body, to_bytes(host))
-            self.assertEqual(request.headers.get('Host'), to_bytes(host))
+            self.assertEqual(response.body, host.encode())
+            self.assertEqual(request.headers.get('Host'), host.encode())
 
         request = Request(self.getURL('host'), headers={'Host': host})
         return self.download_request(request, Spider('foo')).addCallback(_test)
@@ -522,49 +521,6 @@ class Https11TestCase(Http11TestCase):
             yield download_handler.close()
 
 
-class Https2TestCase(Https11TestCase):
-    scheme = 'https'
-    download_handler_cls = H2DownloadHandler
-    HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
-
-    @defer.inlineCallbacks
-    def test_download_with_maxsize_very_large_file(self):
-        with mock.patch('scrapy.core.http2.stream.logger') as logger:
-            request = Request(self.getURL('largechunkedfile'))
-
-            def check(logger):
-                logger.error.assert_called_once_with(mock.ANY)
-
-            d = self.download_request(request, Spider('foo', download_maxsize=1500))
-            yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
-
-            # As the error message is logged in the dataReceived callback, we
-            # have to give a bit of time to the reactor to process the queue
-            # after closing the connection.
-            d = defer.Deferred()
-            d.addCallback(check)
-            reactor.callLater(.1, d.callback, logger)
-            yield d
-
-    def test_download_broken_content_cause_data_loss(self, url='broken'):
-        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
-
-    def test_download_broken_chunked_content_cause_data_loss(self):
-        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
-
-    def test_download_broken_content_allow_data_loss(self, url='broken'):
-        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
-
-    def test_download_broken_chunked_content_allow_data_loss(self):
-        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
-
-    def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
-        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
-
-    def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
-        raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
-
-
 class Https11WrongHostnameTestCase(Http11TestCase):
     scheme = 'https'
 
@@ -577,23 +533,6 @@ class Https11WrongHostnameTestCase(Http11TestCase):
     certfile = 'keys/example-com.cert.pem'
 
 
-class Https2WrongHostnameTestCase(Https2TestCase):
-    tls_log_message = (
-        'SSL connection certificate: issuer "/C=XW/ST=XW/L=The '
-        'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", '
-        'subject "/C=XW/ST=XW/L=The '
-        'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"'
-    )
-
-    # above tests use a server certificate for "localhost",
-    # client connection to "localhost" too.
-    # here we test that even if the server certificate is for another domain,
-    # "www.example.com" in this case,
-    # the tests still pass
-    keyfile = 'keys/example-com.key.pem'
-    certfile = 'keys/example-com.cert.pem'
-
-
 class Https11InvalidDNSId(Https11TestCase):
     """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
 
@@ -602,14 +541,6 @@ class Https11InvalidDNSId(Https11TestCase):
         self.host = '127.0.0.1'
 
 
-class Https2InvalidDNSId(Https2TestCase):
-    """Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
-
-    def setUp(self):
-        super(Https2InvalidDNSId, self).setUp()
-        self.host = '127.0.0.1'
-
-
 class Https11InvalidDNSPattern(Https11TestCase):
     """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
 
@@ -628,24 +559,6 @@ class Https11InvalidDNSPattern(Https11TestCase):
         super(Https11InvalidDNSPattern, self).setUp()
 
 
-class Https2InvalidDNSPattern(Https2TestCase):
-    """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
-
-    keyfile = 'keys/localhost.ip.key'
-    certfile = 'keys/localhost.ip.crt'
-
-    def setUp(self):
-        try:
-            from service_identity.exceptions import CertificateError  # noqa: F401
-        except ImportError:
-            raise unittest.SkipTest("cryptography lib is too old")
-        self.tls_log_message = (
-            'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
-            'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
-        )
-        super(Https2InvalidDNSPattern, self).setUp()
-
-
 class Https11CustomCiphers(unittest.TestCase):
     scheme = 'https'
     download_handler_cls = HTTP11DownloadHandler
@@ -686,11 +599,6 @@ class Https11CustomCiphers(unittest.TestCase):
         return d
 
 
-class Https2CustomCiphers(Https11CustomCiphers):
-    scheme = 'https'
-    download_handler_cls = H2DownloadHandler
-
-
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
 
@@ -742,10 +650,6 @@ class Http11MockServerTestCase(unittest.TestCase):
         self.assertTrue(reason, 'finished')
 
 
-class Http2MockServerTestCase(Http11MockServerTestCase):
-    """HTTP 2.0 test case with MockServer"""
-
-
 class UriResource(resource.Resource):
     """Return the full uri that was requested"""
 
@@ -837,47 +741,6 @@ class Http11ProxyTestCase(HttpProxyTestCase):
         self.assertIn(domain, timeout.osError)
 
 
-class Https2ProxyTestCase(Http11ProxyTestCase):
-    # only used for HTTPS tests
-    keyfile = 'keys/localhost.key'
-    certfile = 'keys/localhost.crt'
-
-    scheme = 'https'
-    host = u'127.0.0.1'
-
-    download_handler_cls = H2DownloadHandler
-    expected_http_proxy_request_body = b'/'
-
-    def setUp(self):
-        site = server.Site(UriResource(), timeout=None)
-        self.port = reactor.listenSSL(
-            0, site,
-            ssl_context_factory(self.keyfile, self.certfile),
-            interface=self.host
-        )
-        self.portno = self.port.getHost().port
-        self.download_handler = create_instance(self.download_handler_cls, None, get_crawler())
-        self.download_request = self.download_handler.download_request
-
-    def getURL(self, path):
-        return f"{self.scheme}://{self.host}:{self.portno}/{path}"
-
-    def test_download_with_proxy_https_noconnect(self):
-        def _test(response):
-            self.assertEqual(response.status, 200)
-            self.assertEqual(response.url, request.url)
-            self.assertEqual(response.body, b'/')
-
-        http_proxy = '%s?noconnect' % self.getURL('')
-        request = Request('https://example.com', meta={'proxy': http_proxy})
-        with self.assertWarnsRegex(
-            Warning,
-            r'Using HTTPS proxies in the noconnect mode is not supported by the '
-            r'downloader handler.'
-        ):
-            return self.download_request(request, Spider('foo')).addCallback(_test)
-
-
 class HttpDownloadHandlerMock:
 
     def __init__(self, *args, **kwargs):

From f9f008e935c5c892c2839d354ded3ee213057d9e Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sun, 16 Aug 2020 17:04:40 +0530
Subject: [PATCH 097/716] test: add typing-extensions

---
 .travis.yml | 2 +-
 setup.py    | 1 +
 tox.ini     | 4 +++-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index e273e358d..0b55cda19 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,7 +3,7 @@ dist: xenial
 branches:
   only:
     - master
-    - http2  # Remove once merged into master
+    - http2  # ToDo: Remove once merged into master
     - /^\d\.\d+$/
     - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:
diff --git a/setup.py b/setup.py
index 7f4ff0095..c8733ae96 100644
--- a/setup.py
+++ b/setup.py
@@ -33,6 +33,7 @@ install_requires = [
     'protego>=0.1.15',
     'itemadapter>=0.1.0',
     'h2>=3.2.0',
+    'typing-extensions>=3.7',
 ]
 extras_require = {}
 
diff --git a/tox.ini b/tox.ini
index e8fdbd85d..3bf2224ec 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,6 +12,7 @@ deps =
     -ctests/constraints.txt
     -rtests/requirements-py3.txt
     # Extras
+    Twisted[http2]>=17.9.0
     boto3>=1.13.0
     botocore>=1.3.23
     Pillow>=3.4.2
@@ -64,6 +65,7 @@ deps =
     -ctests/constraints.txt
     cryptography==2.0
     cssselect==0.9.1
+    h2==3.2.0
     itemadapter==0.1.0
     parsel==1.5.0
     Protego==0.1.15
@@ -72,12 +74,12 @@ deps =
     queuelib==1.4.2
     service_identity==16.0.0
     Twisted[http2]==17.9.0
+    typing-extensions==3.7
     w3lib==1.17.0
     zope.interface==4.1.3
     -rtests/requirements-py3.txt
     # Extras
     botocore==1.3.23
-    h2==3.2.0
     google-cloud-storage==1.29.0
     Pillow==3.4.2
 

From 38d361792c02ae2b25323258d070c04d8906495a Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sun, 16 Aug 2020 17:55:16 +0530
Subject: [PATCH 098/716] fix: typing & pylint errors

- Ignore typing check for http2 test files
---
 scrapy/core/downloader/handlers/http2.py | 4 ++--
 scrapy/core/http2/protocol.py            | 2 +-
 setup.cfg                                | 6 ++++++
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 650af9778..f2ed40f9b 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -71,8 +71,8 @@ class ScrapyH2Agent:
 
     def __init__(
         self, context_factory,
-        connect_timeout=10,
-        bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None,
+        pool: H2ConnectionPool,
+        connect_timeout=10, bind_address: Optional[bytes] = None,
         crawler=None
     ) -> None:
         self._context_factory = context_factory
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 1ce8b6548..fee391af6 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -323,7 +323,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             elif isinstance(event, SettingsAcknowledged):
                 self.settings_acknowledged(event)
             elif isinstance(event, UnknownFrameReceived):
-                logger.debug(f'UnknownFrameReceived: frame={event.frame}')
+                logger.debug('UnknownFrameReceived: frame={}'.format(event.frame))
 
     # Event handler functions starts here
     def connection_terminated(self, event: ConnectionTerminated) -> None:
diff --git a/setup.cfg b/setup.cfg
index f8e7c0c91..8b70a0e60 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -100,6 +100,9 @@ ignore_errors = True
 [mypy-tests.test_downloader_handlers]
 ignore_errors = True
 
+[mypy-tests.test_download_handlers_http2]
+ignore_errors = True
+
 [mypy-tests.test_engine]
 ignore_errors = True
 
@@ -109,6 +112,9 @@ ignore_errors = True
 [mypy-tests.test_http_request]
 ignore_errors = True
 
+[mypy-tests.test_http2_client_protocol]
+ignore_errors = True
+
 [mypy-tests.test_linkextractors]
 ignore_errors = True
 

From 75fe3d13657e11bdabb9b26b452c6cdacecffec7 Mon Sep 17 00:00:00 2001
From: adityaa30 <k.aditya00@gmail.com>
Date: Mon, 17 Aug 2020 03:47:17 +0530
Subject: [PATCH 099/716] fix: increase timeout to 0.5 seconds

- In Windows specifically the reactor was left unclean by the
  HostnameEndpoint due to the tearDown method of
  test_downloader_handlers.py::HttpTestCase due to
  which the following 2 tests were failing:
  1. test_timeout_download_from_spider_server_hangs
  2. test_timeout_download_from_spider_nodata_rcvd
- Increasing the timeout fixed the test (in local)
---
 setup.cfg                                                     | 2 +-
 tests/test_downloader_handlers.py                             | 4 ++--
 ...ad_handlers_http2.py => test_downloader_handlers_http2.py} | 0
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename tests/{test_download_handlers_http2.py => test_downloader_handlers_http2.py} (100%)

diff --git a/setup.cfg b/setup.cfg
index 8b70a0e60..5b267c295 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -100,7 +100,7 @@ ignore_errors = True
 [mypy-tests.test_downloader_handlers]
 ignore_errors = True
 
-[mypy-tests.test_download_handlers_http2]
+[mypy-tests.test_downloader_handlers_http2]
 ignore_errors = True
 
 [mypy-tests.test_engine]
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index ecac47d90..2b3fa2aca 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -287,7 +287,7 @@ class HttpTestCase(unittest.TestCase):
     def test_timeout_download_from_spider_nodata_rcvd(self):
         # client connects but no data is received
         spider = Spider('foo')
-        meta = {'download_timeout': 0.2}
+        meta = {'download_timeout': 0.5}
         request = Request(self.getURL('wait'), meta=meta)
         d = self.download_request(request, spider)
         yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
@@ -296,7 +296,7 @@ class HttpTestCase(unittest.TestCase):
     def test_timeout_download_from_spider_server_hangs(self):
         # client connects, server send headers and some body bytes but hangs
         spider = Spider('foo')
-        meta = {'download_timeout': 0.2}
+        meta = {'download_timeout': 0.5}
         request = Request(self.getURL('hang-after-headers'), meta=meta)
         d = self.download_request(request, spider)
         yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
diff --git a/tests/test_download_handlers_http2.py b/tests/test_downloader_handlers_http2.py
similarity index 100%
rename from tests/test_download_handlers_http2.py
rename to tests/test_downloader_handlers_http2.py

From a87ab71d1061585e41864d7283557bbe9823a91b Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Tue, 18 Aug 2020 04:47:09 +0530
Subject: [PATCH 100/716] refactor(http2): metadata for Stream

- Add Note about HTTP/2 Cleartext not supported in settings.rst
---
 docs/topics/settings.rst          |  7 +++
 scrapy/core/http2/protocol.py     |  2 +-
 scrapy/core/http2/stream.py       | 73 +++++++++++++++----------------
 scrapy/core/http2/types.py        | 23 ++++++++++
 setup.py                          |  2 +-
 tests/test_downloader_handlers.py |  2 +-
 6 files changed, 68 insertions(+), 41 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index d4d4f9332..0dad30b29 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -627,6 +627,13 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
         'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
     }
 
+.. note::
+
+    Scrapy currently does not support HTTP/2 Cleartext (h2c) since none
+    of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_).
+
+.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
+
 .. setting:: DOWNLOAD_TIMEOUT
 
 DOWNLOAD_TIMEOUT
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index fee391af6..7a3156541 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -289,7 +289,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             self._conn_lost_deferred.callback(self._conn_lost_errors)
 
         for stream in self.streams.values():
-            if stream.request_sent:
+            if stream.metadata['request_sent']:
                 close_reason = StreamCloseReason.CONNECTION_LOST
             else:
                 close_reason = StreamCloseReason.INACTIVE
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 1e136fbd5..bddf50a56 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -5,14 +5,14 @@ from typing import List, Optional, Tuple, TYPE_CHECKING
 from urllib.parse import urlparse
 
 from h2.errors import ErrorCodes
-from h2.exceptions import H2Error, StreamClosedError
+from h2.exceptions import H2Error, StreamClosedError, ProtocolError
 from hpack import HeaderTuple
 from twisted.internet.defer import Deferred, CancelledError
 from twisted.internet.error import ConnectionClosed
 from twisted.python.failure import Failure
 from twisted.web.client import ResponseFailed
 
-from scrapy.core.http2.types import H2ResponseDict
+from scrapy.core.http2.types import H2ResponseDict, H2StreamMetadataDict
 from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
@@ -100,24 +100,14 @@ class Stream:
         self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
         self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
 
-        self.request_start_time = None
-
-        self.content_length = 0 if self._request.body is None else len(self._request.body)
-
-        # Flag to keep track whether this stream has initiated the request
-        self.request_sent = False
-
-        # Flag to track whether we have logged about exceeding download warnsize
-        self._reached_warnsize = False
-
-        # Each time we send a data frame, we will decrease value by the amount send.
-        self.remaining_content_length = self.content_length
-
-        # Flag to keep track whether we have closed this stream
-        self.stream_closed_local = False
-
-        # Flag to keep track whether the server has closed the stream
-        self.stream_closed_server = False
+        self.metadata: H2StreamMetadataDict = {
+            'request_content_length': 0 if self._request.body is None else len(self._request.body),
+            'request_sent': False,
+            'reached_warnsize': False,
+            'remaining_content_length': 0 if self._request.body is None else len(self._request.body),
+            'stream_closed_local': False,
+            'stream_closed_server': False,
+        }
 
         # Private variable used to build the response
         # this response is then converted to appropriate Response class
@@ -132,7 +122,7 @@ class Stream:
             # Close this stream as gracefully as possible
             # If the associated request is initiated we reset this stream
             # else we directly call close() method
-            if self.request_sent:
+            if self.metadata['request_sent']:
                 self.reset_stream(StreamCloseReason.CANCELLED)
             else:
                 self.close(StreamCloseReason.CANCELLED)
@@ -160,7 +150,7 @@ class Stream:
                 self._response['flow_controlled_size'] > self._download_warnsize
                 or content_length_header > self._download_warnsize
             )
-            and not self._reached_warnsize
+            and not self.metadata['reached_warnsize']
         )
 
     def get_response(self) -> Deferred:
@@ -220,7 +210,7 @@ class Stream:
         if self.check_request_url():
             headers = self._get_request_headers()
             self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False)
-            self.request_sent = True
+            self.metadata['request_sent'] = True
             self.send_data()
         else:
             # Close this stream calling the response errback
@@ -238,7 +228,7 @@ class Stream:
             and has initiated request already by sending HEADER frame. If not then
             stream will raise ProtocolError (raise by h2 state machine).
          """
-        if self.stream_closed_local:
+        if self.metadata['stream_closed_local']:
             raise StreamClosedError(self.stream_id)
 
         # Firstly, check what the flow control window is for current stream.
@@ -249,24 +239,24 @@ class Stream:
 
         # We will send no more than the window size or the remaining file size
         # of data in this call, whichever is smaller.
-        bytes_to_send_size = min(window_size, self.remaining_content_length)
+        bytes_to_send_size = min(window_size, self.metadata['remaining_content_length'])
 
         # We now need to send a number of data frames.
         while bytes_to_send_size > 0:
             chunk_size = min(bytes_to_send_size, max_frame_size)
 
-            data_chunk_start_id = self.content_length - self.remaining_content_length
+            data_chunk_start_id = self.metadata['request_content_length'] - self.metadata['remaining_content_length']
             data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size]
 
             self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False)
 
             bytes_to_send_size = bytes_to_send_size - chunk_size
-            self.remaining_content_length = self.remaining_content_length - chunk_size
+            self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size
 
-        self.remaining_content_length = max(0, self.remaining_content_length)
+        self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length'])
 
         # End the stream if no more data needs to be send
-        if self.remaining_content_length == 0:
+        if self.metadata['remaining_content_length'] == 0:
             self._protocol.conn.end_stream(self.stream_id)
 
         # Q. What about the rest of the data?
@@ -277,7 +267,11 @@ class Stream:
         Send data that earlier could not be sent as we were
         blocked behind the flow control.
         """
-        if self.remaining_content_length and not self.stream_closed_server and self.request_sent:
+        if (
+            self.metadata['remaining_content_length']
+            and not self.metadata['stream_closed_server']
+            and self.metadata['request_sent']
+        ):
             self.send_data()
 
     def receive_data(self, data: bytes, flow_controlled_length: int) -> None:
@@ -290,7 +284,7 @@ class Stream:
             return
 
         if self._log_warnsize:
-            self._reached_warnsize = True
+            self.metadata['reached_warnsize'] = True
             warning_msg = (
                 f'Received more ({self._response["flow_controlled_size"]}) bytes than download '
                 f'warn size ({self._download_warnsize}) in request {self._request}'
@@ -314,7 +308,7 @@ class Stream:
             return
 
         if self._log_warnsize:
-            self._reached_warnsize = True
+            self.metadata['reached_warnsize'] = True
             warning_msg = (
                 f'Expected response size ({expected_size}) larger than '
                 f'download warn size ({self._download_warnsize}) in request {self._request}'
@@ -323,18 +317,18 @@ class Stream:
 
     def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None:
         """Close this stream by sending a RST_FRAME to the remote peer"""
-        if self.stream_closed_local:
+        if self.metadata['stream_closed_local']:
             raise StreamClosedError(self.stream_id)
 
         # Clear buffer earlier to avoid keeping data in memory for a long time
         self._response['body'].truncate(0)
 
-        self.stream_closed_local = True
+        self.metadata['stream_closed_local'] = True
         self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
         self.close(reason)
 
     def _is_data_lost(self) -> bool:
-        assert self.stream_closed_server
+        assert self.metadata['stream_closed_server']
 
         expected_size = self._response['flow_controlled_size']
         received_body_size = int(self._response['headers'][b'Content-Length'])
@@ -349,7 +343,7 @@ class Stream:
     ) -> None:
         """Based on the reason sent we will handle each case.
         """
-        if self.stream_closed_server:
+        if self.metadata['stream_closed_server']:
             raise StreamClosedError(self.stream_id)
 
         if not isinstance(reason, StreamCloseReason):
@@ -362,7 +356,7 @@ class Stream:
         if not from_protocol:
             self._protocol.pop_stream(self.stream_id)
 
-        self.stream_closed_server = True
+        self.metadata['stream_closed_server'] = True
 
         # We do not check for Content-Length or Transfer-Encoding in response headers
         # and add `partial` flag as in HTTP/1.1 as 'A request or response that includes
@@ -402,7 +396,10 @@ class Stream:
 
         elif reason is StreamCloseReason.RESET:
             self._deferred_response.errback(ResponseFailed([
-                Failure(f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM')
+                Failure(
+                    f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM',
+                    ProtocolError
+                )
             ]))
 
         elif reason is StreamCloseReason.CONNECTION_LOST:
diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py
index d2aa1a9d8..ff8d94066 100644
--- a/scrapy/core/http2/types.py
+++ b/scrapy/core/http2/types.py
@@ -31,6 +31,29 @@ class H2ConnectionMetadataDict(TypedDict):
     default_download_warnsize: int
 
 
+class H2StreamMetadataDict(TypedDict):
+    """Metadata of an HTTP/2 connection stream
+    initialized when stream is instantiated
+    """
+
+    request_content_length: int
+
+    # Flag to keep track whether the stream has initiated the request
+    request_sent: bool
+
+    # Flag to track whether we have logged about exceeding download warnsize
+    reached_warnsize: bool
+
+    # Each time we send a data frame, we will decrease value by the amount send.
+    remaining_content_length: int
+
+    # Flag to keep track whether we have closed this stream
+    stream_closed_local: bool
+
+    # Flag to keep track whether the server has closed the stream
+    stream_closed_server: bool
+
+
 class H2ResponseDict(TypedDict):
     # Data received frame by frame from the server is appended
     # and passed to the response Deferred when completely received.
diff --git a/setup.py b/setup.py
index c8733ae96..66f369a71 100644
--- a/setup.py
+++ b/setup.py
@@ -97,4 +97,4 @@ setup(
     python_requires='>=3.5.2',
     install_requires=install_requires,
     extras_require=extras_require,
-)
\ No newline at end of file
+)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 2b3fa2aca..e3777ee1d 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -232,7 +232,7 @@ class HttpTestCase(unittest.TestCase):
         r.putChild(b"echo", Echo())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
-        self.host = u'localhost'
+        self.host = 'localhost'
         if self.scheme == 'https':
             # Using WrappingFactory do not enable HTTP/2 failing all the
             # tests with H2DownloadHandler

From a206ac5f6f4e5eb057ecc535556a15087b250d40 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Tue, 18 Aug 2020 07:36:00 +0530
Subject: [PATCH 101/716] tests: disable python 3.5 for travis and azure

---
 .travis.yml         | 25 +++++++++++++------------
 azure-pipelines.yml |  7 ++++---
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0b55cda19..6628e8e43 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,18 +18,19 @@ matrix:
       python: 3.7  # Keep in sync with .readthedocs.yml
     - env: TOXENV=typing
       python: 3.8
-
-    - env: TOXENV=pinned
-      python: 3.5.2
-    - env: TOXENV=asyncio-pinned
-      python: 3.5.2  # We use additional code to support 3.5.3 and earlier
-    - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0
-
-    - env: TOXENV=py
-      python: 3.5
-    - env: TOXENV=asyncio
-      python: 3.5  # We use specific code to support >= 3.5.4, < 3.6
-    - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
+  
+# ToDo: Remove once merged into master
+#    - env: TOXENV=pinned
+#      python: 3.5.2
+#    - env: TOXENV=asyncio-pinned
+#      python: 3.5.2  # We use additional code to support 3.5.3 and earlier
+#    - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0
+#
+#    - env: TOXENV=py
+#      python: 3.5
+#    - env: TOXENV=asyncio
+#      python: 3.5  # We use specific code to support >= 3.5.4, < 3.6
+#    - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
 
     - env: TOXENV=py
       python: 3.6
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 710e42090..c77c128b3 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -4,9 +4,10 @@ pool:
   vmImage: 'windows-latest'
 strategy:
   matrix:
-    Python35:
-      python.version: '3.5'
-      TOXENV: windows-pinned
+# ToDo: Remove once merged into master
+#    Python35:
+#      python.version: '3.5'
+#      TOXENV: windows-pinned
     Python36:
       python.version: '3.6'
     Python37:

From e3233b79deee6ad283ed4316135e30ab774c1078 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 19 Aug 2020 05:10:19 +0530
Subject: [PATCH 102/716] refactor(h2-stream): alphabetical order of imports

---
 scrapy/core/downloader/handlers/http2.py | 2 +-
 scrapy/core/http2/stream.py              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index f2ed40f9b..ddd813cec 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -29,7 +29,7 @@ class H2DownloadHandler:
     def from_crawler(cls, crawler):
         return cls(crawler.settings, crawler)
 
-    def download_request(self, request: Request, spider: Spider):
+    def download_request(self, request: Request, spider: Spider) -> Deferred:
         agent = ScrapyH2Agent(
             context_factory=self._context_factory,
             pool=self._pool,
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index bddf50a56..33302421e 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -5,7 +5,7 @@ from typing import List, Optional, Tuple, TYPE_CHECKING
 from urllib.parse import urlparse
 
 from h2.errors import ErrorCodes
-from h2.exceptions import H2Error, StreamClosedError, ProtocolError
+from h2.exceptions import H2Error, ProtocolError, StreamClosedError
 from hpack import HeaderTuple
 from twisted.internet.defer import Deferred, CancelledError
 from twisted.internet.error import ConnectionClosed

From 30eb005639b77ea94e8859e1d041dfe53048cf77 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 19 Aug 2020 06:25:04 +0530
Subject: [PATCH 103/716] fix: InvalidNegotiatedProtocol __str__ method

---
 scrapy/core/http2/agent.py    | 2 +-
 scrapy/core/http2/protocol.py | 2 +-
 scrapy/core/http2/stream.py   | 3 +++
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index f4ac29bc6..f829cc5f8 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -68,7 +68,7 @@ class H2ConnectionPool:
 
         # Now as we have established a proper HTTP/2 connection
         # we fire all the deferred's with the connection instance
-        pending_requests = self._pending_requests.pop(key)
+        pending_requests = self._pending_requests.pop(key, None)
         while pending_requests:
             d = pending_requests.popleft()
             d.callback(conn)
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 7a3156541..c6f00423a 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -39,7 +39,7 @@ class InvalidNegotiatedProtocol(H2Error):
         self.negotiated_protocol = negotiated_protocol
 
     def __str__(self) -> str:
-        return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}'
+        return f'InvalidNegotiatedProtocol: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}'
 
 
 class RemoteTerminatedConnection(H2Error):
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 33302421e..df7470e11 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -31,6 +31,9 @@ class InactiveStreamClosed(ConnectionClosed):
     def __init__(self, request: Request):
         self.request = request
 
+    def __str__(self) -> str:
+        return f'InactiveStreamClosed: Connection was closed without sending the request {self.request!r}'
+
 
 class InvalidHostname(H2Error):
 

From 2f00666d749b4a736536bafc786e63be06113676 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 19 Aug 2020 07:31:52 +0530
Subject: [PATCH 104/716] refactor: move agents & context-factory

---
 scrapy/core/downloader/contextfactory.py | 22 ++++++++++++-
 scrapy/core/downloader/handlers/http2.py | 31 ++---------------
 scrapy/core/http2/agent.py               | 42 ++++++++++++++----------
 3 files changed, 49 insertions(+), 46 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 5768d8f8e..073ef16bf 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,10 +1,12 @@
 import warnings
 
 from OpenSSL import SSL
+from twisted.internet._sslverify import _setAcceptableProtocols
 from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
 from twisted.web.client import BrowserLikePolicyForHTTPS
 from twisted.web.iweb import IPolicyForHTTPS
 from zope.interface.declarations import implementer
+from zope.interface.verify import verifyObject
 
 from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions
 from scrapy.utils.misc import create_instance, load_object
@@ -84,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
     The default OpenSSL method is ``TLS_METHOD`` (also called
     ``SSLv23_METHOD``) which allows TLS protocol negotiation.
     """
-    def creatorForNetloc(self, hostname, port):
 
+    def creatorForNetloc(self, hostname, port):
         # trustRoot set to platformTrust() will use the platform's root CAs.
         #
         # This means that a website like https://www.cacert.org will be rejected
@@ -97,6 +99,24 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
         )
 
 
+@implementer(IPolicyForHTTPS)
+class AcceptableProtocolsContextFactory:
+    """Context factory to used to override the acceptable protocols
+    to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN
+    negotiation.
+    """
+
+    def __init__(self, context_factory, acceptable_protocols):
+        verifyObject(IPolicyForHTTPS, context_factory)
+        self._wrapped_context_factory = context_factory
+        self._acceptable_protocols = acceptable_protocols
+
+    def creatorForNetloc(self, hostname, port):
+        options = self._wrapped_context_factory.creatorForNetloc(hostname, port)
+        _setAcceptableProtocols(options._ctx, self._acceptable_protocols)
+        return options
+
+
 def load_context_factory_from_settings(settings, crawler):
     ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
     context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index ddd813cec..4be888bda 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -1,16 +1,15 @@
 import warnings
 from time import time
-from typing import Optional, Tuple
+from typing import Optional
 from urllib.parse import urldefrag
 
-from twisted.internet.base import ReactorBase
 from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
-from twisted.web.client import BrowserLikePolicyForHTTPS, URI
+from twisted.web.client import URI
 
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
-from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
+from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
@@ -41,30 +40,6 @@ class H2DownloadHandler:
         self._pool.close_connections()
 
 
-class ScrapyProxyH2Agent(H2Agent):
-    def __init__(
-        self, reactor: ReactorBase,
-        proxy_uri: URI, pool: H2ConnectionPool,
-        context_factory=BrowserLikePolicyForHTTPS(),
-        connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
-    ) -> None:
-        super(ScrapyProxyH2Agent, self).__init__(
-            reactor=reactor,
-            pool=pool,
-            context_factory=context_factory,
-            connect_timeout=connect_timeout,
-            bind_address=bind_address
-        )
-        self._proxy_uri = proxy_uri
-
-    def get_endpoint(self, uri: URI):
-        return self.endpoint_factory.endpointForURI(self._proxy_uri)
-
-    def get_key(self, uri: URI) -> Tuple:
-        """We use the proxy uri instead of uri obtained from request url"""
-        return "http-proxy", self._proxy_uri.host, self._proxy_uri.port
-
-
 class ScrapyH2Agent:
     _Agent = H2Agent
     _ProxyAgent = ScrapyProxyH2Agent
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index f829cc5f8..d950c6cfb 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -2,17 +2,14 @@ from collections import deque
 from typing import Deque, Dict, List, Tuple, Optional
 
 from twisted.internet import defer
-from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions
 from twisted.internet.base import ReactorBase
 from twisted.internet.defer import Deferred
 from twisted.internet.endpoints import HostnameEndpoint
 from twisted.python.failure import Failure
 from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
 from twisted.web.error import SchemeNotSupported
-from twisted.web.iweb import IPolicyForHTTPS
-from zope.interface import implementer
-from zope.interface.verify import verifyObject
 
+from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
 from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
 from scrapy.http.request import Request
 from scrapy.settings import Settings
@@ -96,19 +93,6 @@ class H2ConnectionPool:
             conn.transport.abortConnection()
 
 
-@implementer(IPolicyForHTTPS)
-class AcceptableProtocolsContextFactory:
-    def __init__(self, context_factory, acceptable_protocols: List[bytes]) -> None:
-        verifyObject(IPolicyForHTTPS, context_factory)
-        self._wrapped_context_factory = context_factory
-        self._acceptable_protocols = acceptable_protocols
-
-    def creatorForNetloc(self, hostname, port) -> ClientTLSOptions:
-        options = self._wrapped_context_factory.creatorForNetloc(hostname, port)
-        _setAcceptableProtocols(options._ctx, self._acceptable_protocols)
-        return options
-
-
 class H2Agent:
     def __init__(
         self, reactor: ReactorBase, pool: H2ConnectionPool,
@@ -144,3 +128,27 @@ class H2Agent:
         d = self._pool.get_connection(key, uri, endpoint)
         d.addCallback(lambda conn: conn.request(request, spider))
         return d
+
+
+class ScrapyProxyH2Agent(H2Agent):
+    def __init__(
+        self, reactor: ReactorBase,
+        proxy_uri: URI, pool: H2ConnectionPool,
+        context_factory=BrowserLikePolicyForHTTPS(),
+        connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
+    ) -> None:
+        super(ScrapyProxyH2Agent, self).__init__(
+            reactor=reactor,
+            pool=pool,
+            context_factory=context_factory,
+            connect_timeout=connect_timeout,
+            bind_address=bind_address
+        )
+        self._proxy_uri = proxy_uri
+
+    def get_endpoint(self, uri: URI):
+        return self.endpoint_factory.endpointForURI(self._proxy_uri)
+
+    def get_key(self, uri: URI) -> Tuple:
+        """We use the proxy uri instead of uri obtained from request url"""
+        return "http-proxy", self._proxy_uri.host, self._proxy_uri.port

From e90be0d8a5e3c20ab6aced22ccac59a876db8c99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 21 Aug 2020 14:09:52 +0200
Subject: [PATCH 105/716] Mark the new test as xfail for xmliter_lxml

---
 tests/test_utils_iterators.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index bbdc88dd1..ae64e36cf 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,5 +1,6 @@
 import os
 
+from pytest import mark
 from twisted.trial import unittest
 
 from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
@@ -149,6 +150,26 @@ class XmliterTestCase(unittest.TestCase):
         self.assertEqual(node.xpath('id/text()').getall(), [])
         self.assertEqual(node.xpath('price/text()').getall(), [])
 
+    def test_xmliter_namespaced_nodename(self):
+        body = b"""
+            <?xml version="1.0" encoding="UTF-8"?>
+            <rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
+                <channel>
+                <title>My Dummy Company</title>
+                <link>http://www.mydummycompany.com</link>
+                <description>This is a dummy company. We do nothing.</description>
+                <item>
+                    <title>Item 1</title>
+                    <description>This is item 1</description>
+                    <link>http://www.mydummycompany.com/items/1</link>
+                    <g:image_link>http://www.mydummycompany.com/images/item1.jpg</g:image_link>
+                    <g:id>ITEM_1</g:id>
+                    <g:price>400</g:price>
+                </item>
+                </channel>
+            </rss>
+        """
+        response = XmlResponse(url='http://mydummycompany.com', body=body)
         my_iter = self.xmliter(response, 'g:image_link')
         node = next(my_iter)
         node.register_namespace('g', 'http://base.google.com/ns/1.0')
@@ -187,6 +208,10 @@ class XmliterTestCase(unittest.TestCase):
 class LxmlXmliterTestCase(XmliterTestCase):
     xmliter = staticmethod(xmliter_lxml)
 
+    @mark.xfail(reason='known bug of the current implementation')
+    def test_xmliter_namespaced_nodename(self):
+        super().test_xmliter_namespaced_nodename()
+
     def test_xmliter_iterate_namespace(self):
         body = b"""
             <?xml version="1.0" encoding="UTF-8"?>

From afd3a4d116809ecf4c334657c5150c157aa9465b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 21 Aug 2020 17:04:02 +0200
Subject: [PATCH 106/716] Fix style issue

---
 scrapy/utils/iterators.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index c356ad7f8..6f6b5e337 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -41,7 +41,15 @@ def xmliter(obj, nodename):
 
     r = re.compile(r'<%(np)s[\s>].*?</%(np)s>' % {'np': nodename_patt}, re.DOTALL)
     for match in r.finditer(text):
-        nodetext = document_header + match.group().replace(nodename, '%s %s' % (nodename, ' '.join(namespaces.values())), 1) + header_end
+        nodetext = (
+            document_header
+            + match.group().replace(
+                nodename,
+                '%s %s' % (nodename, ' '.join(namespaces.values())),
+                1
+            )
+            + header_end
+        )
         yield Selector(text=nodetext, type='xml')
 
 

From 1432161477673152f4d2f95cd32829a81ffe3f70 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Mon, 24 Aug 2020 15:40:01 +0530
Subject: [PATCH 107/716] fix: bump min typing-extensions version to 3.7.4

- typing-extensions>=3.7.4 only supports TypedDict
---
 setup.py | 2 +-
 tox.ini  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 607f1dadf..39482d383 100644
--- a/setup.py
+++ b/setup.py
@@ -33,7 +33,7 @@ install_requires = [
     'protego>=0.1.15',
     'itemadapter>=0.1.0',
     'h2>=3.2.0',
-    'typing-extensions>=3.7',
+    'typing-extensions>=3.7.4',
 ]
 extras_require = {}
 
diff --git a/tox.ini b/tox.ini
index 3f6fd1224..0a88ed8af 100644
--- a/tox.ini
+++ b/tox.ini
@@ -74,7 +74,7 @@ deps =
     queuelib==1.4.2
     service_identity==16.0.0
     Twisted[http2]==17.9.0
-    typing-extensions==3.7
+    typing-extensions==3.7.4
     w3lib==1.17.0
     zope.interface==4.1.3
     -rtests/requirements-py3.txt

From 2f28cee3ce482a9380c816b689fc6a5dabc60295 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 25 Aug 2020 17:49:17 +0200
Subject: [PATCH 108/716] Add a test to cover searching for a missing node name

---
 tests/test_utils_iterators.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index ae64e36cf..2adccebb8 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -175,6 +175,30 @@ class XmliterTestCase(unittest.TestCase):
         node.register_namespace('g', 'http://base.google.com/ns/1.0')
         self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
 
+    def test_xmliter_namespaced_nodename_missing(self):
+        body = b"""
+            <?xml version="1.0" encoding="UTF-8"?>
+            <rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
+                <channel>
+                <title>My Dummy Company</title>
+                <link>http://www.mydummycompany.com</link>
+                <description>This is a dummy company. We do nothing.</description>
+                <item>
+                    <title>Item 1</title>
+                    <description>This is item 1</description>
+                    <link>http://www.mydummycompany.com/items/1</link>
+                    <g:image_link>http://www.mydummycompany.com/images/item1.jpg</g:image_link>
+                    <g:id>ITEM_1</g:id>
+                    <g:price>400</g:price>
+                </item>
+                </channel>
+            </rss>
+        """
+        response = XmlResponse(url='http://mydummycompany.com', body=body)
+        my_iter = self.xmliter(response, 'g:link_image')
+        with self.assertRaises(StopIteration):
+            next(my_iter)
+
     def test_xmliter_exception(self):
         body = (
             '<?xml version="1.0" encoding="UTF-8"?>'

From 450ba6b51f7cc3cd89a92e8ff4d9e105865eb862 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Wed, 26 Aug 2020 17:20:59 +0530
Subject: [PATCH 109/716] fix(typo): stream -> streams, use isinstance

---
 scrapy/core/http2/protocol.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index c6f00423a..6647ae0b7 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -90,7 +90,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         # all requests in a pool and send them as the connection is made
         self._pending_request_stream_pool: deque = deque()
 
-        # Counter to keep track of opened stream. This counter
+        # Counter to keep track of opened streams. This counter
         # is used to make sure that not more than MAX_CONCURRENT_STREAMS
         # streams are opened which leads to ProtocolError
         # We use simple FIFO policy to handle pending requests
@@ -218,7 +218,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         """We close the connection with InvalidNegotiatedProtocol exception
         when the connection was not made via h2 protocol"""
         negotiated_protocol = self.transport.negotiatedProtocol
-        if type(negotiated_protocol) is bytes:
+        if isinstance(negotiated_protocol, bytes):
             negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8')
         if negotiated_protocol != 'h2':
             # Here we have not initiated the connection yet

From 5e36f539e28631625862b84d8ca1c7c0134584b0 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Thu, 27 Aug 2020 15:12:22 +0530
Subject: [PATCH 110/716] chore: remove typing-extensions dependency

---
 scrapy/core/http2/protocol.py | 52 ++++++++++++++++-----------
 scrapy/core/http2/stream.py   | 27 +++++++++++---
 scrapy/core/http2/types.py    | 67 -----------------------------------
 setup.py                      |  1 -
 tox.ini                       |  1 -
 5 files changed, 55 insertions(+), 93 deletions(-)
 delete mode 100644 scrapy/core/http2/types.py

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 6647ae0b7..0b872f6ba 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -3,7 +3,6 @@ import itertools
 import logging
 from collections import deque
 from ipaddress import IPv4Address, IPv6Address
-from typing import Dict, List, Optional, Union
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
@@ -22,10 +21,10 @@ from twisted.internet.ssl import Certificate
 from twisted.protocols.policies import TimeoutMixin
 from twisted.python.failure import Failure
 from twisted.web.client import URI
+from typing import Dict, List, Optional, Union
 from zope.interface import implementer
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
-from scrapy.core.http2.types import H2ConnectionMetadataDict
 from scrapy.http import Request
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
@@ -90,26 +89,39 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         # all requests in a pool and send them as the connection is made
         self._pending_request_stream_pool: deque = deque()
 
-        # Counter to keep track of opened streams. This counter
-        # is used to make sure that not more than MAX_CONCURRENT_STREAMS
-        # streams are opened which leads to ProtocolError
-        # We use simple FIFO policy to handle pending requests
-        self._active_streams = 0
-
-        # Flag to keep track if settings were acknowledged by the remote
-        # This ensures that we have established a HTTP/2 connection
-        self._settings_acknowledged = False
-
         # Save an instance of errors raised which lead to losing the connection
         # We pass these instances to the streams ResponseFailed() failure
         self._conn_lost_errors: List[BaseException] = []
 
-        self.metadata: H2ConnectionMetadataDict = {
+        # Some meta data of this connection
+        # initialized when connection is successfully made
+        self.metadata: Dict = {
+            # Peer certificate instance
             'certificate': None,
+
+            # Address of the server we are connected to which
+            # is updated when HTTP/2 connection is  made successfully
             'ip_address': None,
+
+            # URI of the peer HTTP/2 connection is made
             'uri': uri,
+
+            # Both ip_address and uri are used by the Stream before
+            # initiating the request to verify that the base address
+
+            # Variables taken from Project Settings
             'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'),
             'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'),
+
+            # Counter to keep track of opened streams. This counter
+            # is used to make sure that not more than MAX_CONCURRENT_STREAMS
+            # streams are opened which leads to ProtocolError
+            # We use simple FIFO policy to handle pending requests
+            'active_streams': 0,
+
+            # Flag to keep track if settings were acknowledged by the remote
+            # This ensures that we have established a HTTP/2 connection
+            'settings_acknowledged': False,
         }
 
     @property
@@ -118,7 +130,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         This is used while initiating pending streams to make sure
         that we initiate stream only during active HTTP/2 Connection
         """
-        return bool(self.transport.connected) and self._settings_acknowledged
+        return bool(self.transport.connected) and self.metadata['settings_acknowledged']
 
     @property
     def allowed_max_concurrent_streams(self) -> int:
@@ -139,10 +151,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         """
         while (
             self._pending_request_stream_pool
-            and self._active_streams < self.allowed_max_concurrent_streams
+            and self.metadata['active_streams'] < self.allowed_max_concurrent_streams
             and self.h2_connected
         ):
-            self._active_streams += 1
+            self.metadata['active_streams'] += 1
             stream = self._pending_request_stream_pool.popleft()
             stream.initiate_request()
             self._write_to_transport()
@@ -151,7 +163,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         """Perform cleanup when a stream is closed
         """
         stream = self.streams.pop(stream_id)
-        self._active_streams -= 1
+        self.metadata['active_streams'] -= 1
         self._send_pending_requests()
         return stream
 
@@ -261,7 +273,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         if (
             self.conn.open_outbound_streams > 0
             or self.conn.open_inbound_streams > 0
-            or self._active_streams > 0
+            or self.metadata['active_streams'] > 0
         ):
             error_code = ErrorCodes.PROTOCOL_ERROR
         else:
@@ -295,7 +307,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
                 close_reason = StreamCloseReason.INACTIVE
             stream.close(close_reason, self._conn_lost_errors, from_protocol=True)
 
-        self._active_streams -= len(self.streams)
+        self.metadata['active_streams'] -= len(self.streams)
         self.streams.clear()
         self._pending_request_stream_pool.clear()
         self.conn.close_connection()
@@ -338,7 +350,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self.streams[event.stream_id].receive_headers(event.headers)
 
     def settings_acknowledged(self, event: SettingsAcknowledged) -> None:
-        self._settings_acknowledged = True
+        self.metadata['settings_acknowledged'] = True
 
         # Send off all the pending requests as now we have
         # established a proper HTTP/2 connection
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index df7470e11..e01e76ae5 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -1,7 +1,6 @@
 import logging
 from enum import Enum
 from io import BytesIO
-from typing import List, Optional, Tuple, TYPE_CHECKING
 from urllib.parse import urlparse
 
 from h2.errors import ErrorCodes
@@ -11,8 +10,9 @@ from twisted.internet.defer import Deferred, CancelledError
 from twisted.internet.error import ConnectionClosed
 from twisted.python.failure import Failure
 from twisted.web.client import ResponseFailed
+from typing import Dict
+from typing import List, Optional, Tuple, TYPE_CHECKING
 
-from scrapy.core.http2.types import H2ResponseDict, H2StreamMetadataDict
 from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
@@ -103,21 +103,40 @@ class Stream:
         self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize)
         self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize)
 
-        self.metadata: H2StreamMetadataDict = {
+        # Metadata of an HTTP/2 connection stream
+        # initialized when stream is instantiated
+        self.metadata: Dict = {
             'request_content_length': 0 if self._request.body is None else len(self._request.body),
+
+            # Flag to keep track whether the stream has initiated the request
             'request_sent': False,
+
+            # Flag to track whether we have logged about exceeding download warnsize
             'reached_warnsize': False,
+
+            # Each time we send a data frame, we will decrease value by the amount send.
             'remaining_content_length': 0 if self._request.body is None else len(self._request.body),
+
+            # Flag to keep track whether client (self) have closed this stream
             'stream_closed_local': False,
+
+            # Flag to keep track whether the server has closed the stream
             'stream_closed_server': False,
         }
 
         # Private variable used to build the response
         # this response is then converted to appropriate Response class
         # passed to the response deferred callback
-        self._response: H2ResponseDict = {
+        self._response: Dict = {
+            # Data received frame by frame from the server is appended
+            # and passed to the response Deferred when completely received.
             'body': BytesIO(),
+
+            # The amount of data received that counts against the
+            # flow control window
             'flow_controlled_size': 0,
+
+            # Headers received after sending the request
             'headers': Headers({}),
         }
 
diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py
deleted file mode 100644
index ff8d94066..000000000
--- a/scrapy/core/http2/types.py
+++ /dev/null
@@ -1,67 +0,0 @@
-from io import BytesIO
-from ipaddress import IPv4Address, IPv6Address
-from typing import Union, Optional
-
-from twisted.internet.ssl import Certificate
-from twisted.web.client import URI
-# for python < 3.8 -- typing.TypedDict is undefined
-from typing_extensions import TypedDict
-
-from scrapy.http.headers import Headers
-
-
-class H2ConnectionMetadataDict(TypedDict):
-    """Some meta data of this connection
-    initialized when connection is successfully made
-    """
-    certificate: Optional[Certificate]
-
-    # Address of the server we are connected to which
-    # is updated when HTTP/2 connection is  made successfully
-    ip_address: Optional[Union[IPv4Address, IPv6Address]]
-
-    # URI of the peer HTTP/2 connection is made
-    uri: URI
-
-    # Both ip_address and uri are used by the Stream before
-    # initiating the request to verify that the base address
-
-    # Variables taken from Project Settings
-    default_download_maxsize: int
-    default_download_warnsize: int
-
-
-class H2StreamMetadataDict(TypedDict):
-    """Metadata of an HTTP/2 connection stream
-    initialized when stream is instantiated
-    """
-
-    request_content_length: int
-
-    # Flag to keep track whether the stream has initiated the request
-    request_sent: bool
-
-    # Flag to track whether we have logged about exceeding download warnsize
-    reached_warnsize: bool
-
-    # Each time we send a data frame, we will decrease value by the amount send.
-    remaining_content_length: int
-
-    # Flag to keep track whether we have closed this stream
-    stream_closed_local: bool
-
-    # Flag to keep track whether the server has closed the stream
-    stream_closed_server: bool
-
-
-class H2ResponseDict(TypedDict):
-    # Data received frame by frame from the server is appended
-    # and passed to the response Deferred when completely received.
-    body: BytesIO
-
-    # The amount of data received that counts against the flow control
-    # window
-    flow_controlled_size: int
-
-    # Headers received after sending the request
-    headers: Headers
diff --git a/setup.py b/setup.py
index 39482d383..34af7ec67 100644
--- a/setup.py
+++ b/setup.py
@@ -33,7 +33,6 @@ install_requires = [
     'protego>=0.1.15',
     'itemadapter>=0.1.0',
     'h2>=3.2.0',
-    'typing-extensions>=3.7.4',
 ]
 extras_require = {}
 
diff --git a/tox.ini b/tox.ini
index 0a88ed8af..78090d6de 100644
--- a/tox.ini
+++ b/tox.ini
@@ -74,7 +74,6 @@ deps =
     queuelib==1.4.2
     service_identity==16.0.0
     Twisted[http2]==17.9.0
-    typing-extensions==3.7.4
     w3lib==1.17.0
     zope.interface==4.1.3
     -rtests/requirements-py3.txt

From a8aedbeb7c20c7e2ec041e49d1567a499cd3acdb Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sat, 29 Aug 2020 12:12:18 +0530
Subject: [PATCH 111/716] chore: rearrange imports

---
 scrapy/core/http2/protocol.py | 2 +-
 scrapy/core/http2/stream.py   | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 0b872f6ba..e32e2b6fe 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -3,6 +3,7 @@ import itertools
 import logging
 from collections import deque
 from ipaddress import IPv4Address, IPv6Address
+from typing import Dict, List, Optional, Union
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
@@ -21,7 +22,6 @@ from twisted.internet.ssl import Certificate
 from twisted.protocols.policies import TimeoutMixin
 from twisted.python.failure import Failure
 from twisted.web.client import URI
-from typing import Dict, List, Optional, Union
 from zope.interface import implementer
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index e01e76ae5..ef90773b6 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -2,6 +2,7 @@ import logging
 from enum import Enum
 from io import BytesIO
 from urllib.parse import urlparse
+from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
 
 from h2.errors import ErrorCodes
 from h2.exceptions import H2Error, ProtocolError, StreamClosedError
@@ -10,8 +11,6 @@ from twisted.internet.defer import Deferred, CancelledError
 from twisted.internet.error import ConnectionClosed
 from twisted.python.failure import Failure
 from twisted.web.client import ResponseFailed
-from typing import Dict
-from typing import List, Optional, Tuple, TYPE_CHECKING
 
 from scrapy.http import Request
 from scrapy.http.headers import Headers

From a8e895e684184e967a751ea28a7102f21dd74834 Mon Sep 17 00:00:00 2001
From: maranqz <ilia.sergunin@gmail.com>
Date: Sun, 30 Aug 2020 10:57:22 +0300
Subject: [PATCH 112/716] kwargs for Item exporters classes test docs

---
 docs/topics/feed-exports.rst    |  5 +++++
 scrapy/extensions/feedexport.py |  1 +
 scrapy/utils/conf.py            |  1 +
 tests/test_feedexport.py        | 37 +++++++++++++++++++++++++++++++++
 4 files changed, 44 insertions(+)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9fb2189e8..e69a64195 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -303,6 +303,9 @@ For instance::
             'store_empty': False,
             'fields': None,
             'indent': 4,
+            'item_export_kwargs': {
+               'export_empty_fields': True,
+            },
         }, 
         '/home/user/documents/items.xml': {
             'format': 'xml',
@@ -332,6 +335,8 @@ as a fallback value if that key is not provided for a specific feed definition:
 
 -   ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
 
+-   ``item_export_kwargs``: dict with kwargs for :ref:`Item exporters <topics-exporters>` classes.
+
 -   ``overwrite``: whether to overwrite the file if it already exists
     (``True``) or append to its content (``False``).
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 980825499..f32d48fa5 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -360,6 +360,7 @@ class FeedExporter:
             fields_to_export=feed_options['fields'],
             encoding=feed_options['encoding'],
             indent=feed_options['indent'],
+            **feed_options['item_export_kwargs'],
         )
         slot = _FeedSlot(
             file=file,
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 4e7a9967e..b904c4a03 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -121,6 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings):
     out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
+    out.setdefault("item_export_kwargs", dict())
     if settings["FEED_EXPORT_INDENT"] is None:
         out.setdefault("indent", None)
     else:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 840e0f87b..e88e4f5ce 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1252,6 +1252,43 @@ class FeedExportTest(FeedExportTestBase):
         for fmt in ['json', 'xml', 'csv']:
             self.assertIn(f'Error storing {fmt} feed (2 items)', str(log))
 
+    @defer.inlineCallbacks
+    def test_extend_kwargs(self):
+        items = [{'foo': 'FOO', 'bar': 'BAR'}]
+
+        expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8')
+        expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8')
+        test_cases = [
+            # with title
+            {
+                'options': {
+                    'format': 'csv',
+                    'item_export_kwargs': dict(include_headers_line=True),
+                },
+                'expected': expected_with_title_csv,
+            },
+            # without title
+            {
+                'options': {
+                    'format': 'csv',
+                    'item_export_kwargs': dict(include_headers_line=False),
+                },
+                'expected': expected_without_title_csv,
+            },
+        ]
+
+        for row in test_cases:
+            feed_options = row['options']
+            settings = {
+                'FEEDS': {
+                    self._random_temp_filename(): feed_options,
+                },
+                'FEED_EXPORT_INDENT': None,
+            }
+
+            data = yield self.exported_data(items, settings)
+            self.assertEqual(row['expected'], data[feed_options['format']])
+
 
 class BatchDeliveriesTest(FeedExportTestBase):
     __test__ = True

From fc3c66ce950e945e7ac6e19fea31e8454147ac29 Mon Sep 17 00:00:00 2001
From: maranqz <ilia.sergunin@gmail.com>
Date: Sun, 30 Aug 2020 11:44:48 +0300
Subject: [PATCH 113/716] fix tests:  *
 FeedExportConfigTestCase.test_feed_complete_default_values_from_settings_empty
  *
 FeedExportConfigTestCase.test_feed_complete_default_values_from_settings_non_empty

---
 tests/test_utils_conf.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 061bc8c7c..dc2560add 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -176,6 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "store_empty": True,
             "uri_params": (1, 2, 3, 4),
             "batch_item_count": 2,
+            "item_export_kwargs": dict(),
         })
 
     def test_feed_complete_default_values_from_settings_non_empty(self):
@@ -198,6 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "store_empty": True,
             "uri_params": None,
             "batch_item_count": 2,
+            "item_export_kwargs": dict(),
         })
 
 

From 71d2c2f1a319b570eb2026707d49c4f62d59296e Mon Sep 17 00:00:00 2001
From: maranqz <ilia.sergunin@gmail.com>
Date: Sun, 30 Aug 2020 12:43:44 +0300
Subject: [PATCH 114/716] improve view of dict

---
 tests/test_feedexport.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e88e4f5ce..5738e36f1 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1263,7 +1263,7 @@ class FeedExportTest(FeedExportTestBase):
             {
                 'options': {
                     'format': 'csv',
-                    'item_export_kwargs': dict(include_headers_line=True),
+                    'item_export_kwargs': {'include_headers_line': True},
                 },
                 'expected': expected_with_title_csv,
             },
@@ -1271,7 +1271,7 @@ class FeedExportTest(FeedExportTestBase):
             {
                 'options': {
                     'format': 'csv',
-                    'item_export_kwargs': dict(include_headers_line=False),
+                    'item_export_kwargs': {'include_headers_line': False},
                 },
                 'expected': expected_without_title_csv,
             },

From eff33a2e7950b29fd69adc40b17b7fe9b0e637c2 Mon Sep 17 00:00:00 2001
From: Aditya <k.aditya00@gmail.com>
Date: Sun, 30 Aug 2020 23:54:43 +0530
Subject: [PATCH 115/716] fix(h2): Mockserver test uses H2DownloadHandler

---
 tests/test_downloader_handlers.py       | 7 ++++---
 tests/test_downloader_handlers_http2.py | 5 +++++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index e3777ee1d..c6f20cb50 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -601,6 +601,7 @@ class Https11CustomCiphers(unittest.TestCase):
 
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
+    settings_dict = None
 
     def setUp(self):
         self.mockserver = MockServer()
@@ -611,7 +612,7 @@ class Http11MockServerTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_download_with_content_length(self):
-        crawler = get_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
         # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid
         # download it
         yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000}))
@@ -620,7 +621,7 @@ class Http11MockServerTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_download(self):
-        crawler = get_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
         yield crawler.crawl(seed=Request(url=self.mockserver.url('')))
         failure = crawler.spider.meta.get('failure')
         self.assertTrue(failure is None)
@@ -629,7 +630,7 @@ class Http11MockServerTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_download_gzip_response(self):
-        crawler = get_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
         body = b'1' * 100  # PayloadResource requires body length to be 100
         request = Request(self.mockserver.url('/payload'), method='POST',
                           body=body, meta={'download_maxsize': 50})
diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index 583dc1d17..253646040 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -110,6 +110,11 @@ class Https2CustomCiphers(Https11CustomCiphers):
 
 class Http2MockServerTestCase(Http11MockServerTestCase):
     """HTTP 2.0 test case with MockServer"""
+    settings_dict = {
+        'DOWNLOAD_HANDLERS': {
+            'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler'
+        }
+    }
 
 
 class Https2ProxyTestCase(Http11ProxyTestCase):

From a41c205928aa2aa86233de0cfb694b0c7ded2297 Mon Sep 17 00:00:00 2001
From: Jose Galdos <jose@bitmaker.la>
Date: Fri, 21 Aug 2020 12:16:37 -0500
Subject: [PATCH 116/716] Update httpstatus documentation.

---
 docs/topics/spider-middleware.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index c6cbdba76..28645fd53 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -255,7 +255,8 @@ this::
 The ``handle_httpstatus_list`` key of :attr:`Request.meta
 <scrapy.http.Request.meta>` can also be used to specify which response codes to
 allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all``
-to ``True`` if you want to allow any response code for a request.
+to ``True`` if you want to allow any response code for a request, and ``False`` to
+disable the effects of the ``handle_httpstatus_all`` key.
 
 Keep in mind, however, that it's usually a bad idea to handle non-200
 responses, unless you really know what you're doing.

From d10464ca96a24d37c854992e2485d622e8eec2b6 Mon Sep 17 00:00:00 2001
From: Ilia Sergunin <ilia.sergunin@gmail.com>
Date: Tue, 1 Sep 2020 10:13:40 +0300
Subject: [PATCH 117/716] Update docs/topics/feed-exports.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/feed-exports.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index e69a64195..1744cfd74 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -335,7 +335,7 @@ as a fallback value if that key is not provided for a specific feed definition:
 
 -   ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
 
--   ``item_export_kwargs``: dict with kwargs for :ref:`Item exporters <topics-exporters>` classes.
+-   ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
 
 -   ``overwrite``: whether to overwrite the file if it already exists
     (``True``) or append to its content (``False``).

From ddc26f3f8fdf766587eda55dfaf60b524eb89e90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 1 Sep 2020 11:26:07 +0200
Subject: [PATCH 118/716] Revert Travis CI changes

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index d9a8e512a..b883c5b78 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,7 +3,6 @@ dist: xenial
 branches:
   only:
     - master
-    - http2  # ToDo: Remove once merged into master
     - /^\d\.\d+$/
     - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:

From 959222df7e80334810636166a8adb60ef35728de Mon Sep 17 00:00:00 2001
From: drs-11 <siddharthadr11@gmail.com>
Date: Sat, 5 Sep 2020 21:32:05 +0530
Subject: [PATCH 119/716] check for unparseable no_proxy values

---
 scrapy/downloadermiddlewares/httpproxy.py    | 7 ++++++-
 tests/test_downloadermiddleware_httpproxy.py | 4 ++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 04da11311..d2665b655 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -13,7 +13,12 @@ class HttpProxyMiddleware:
         self.auth_encoding = auth_encoding
         self.proxies = {}
         for type_, url in getproxies().items():
-            self.proxies[type_] = self._get_proxy(url, type_)
+            try:
+                self.proxies[type_] = self._get_proxy(url, type_)
+            # some values such as '/var/run/docker.sock' can't be parsed
+            # by _parse_proxy and as such should be skipped
+            except ValueError:
+                continue
 
     @classmethod
     def from_crawler(cls, crawler):
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 351631eb8..81d6cc335 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -123,7 +123,11 @@ class TestHttpProxyMiddleware(TestCase):
 
     def test_no_proxy(self):
         os.environ['http_proxy'] = 'https://proxy.for.http:3128'
+        os.environ['no_proxy'] = '/var/run/docker.sock'
         mw = HttpProxyMiddleware()
+        # '/var/run/docker.sock' may be used by the user for
+        # no_proxy value but is not parseable and should be skipped
+        assert 'no' not in mw.proxies
 
         os.environ['no_proxy'] = '*'
         req = Request('http://noproxy.com')

From 7a83474cc534642e8daf581d4b427408343377f2 Mon Sep 17 00:00:00 2001
From: KAILASA's Sri Nithya Priyeshananda
 <68758690+sripriyesha@users.noreply.github.com>
Date: Tue, 8 Sep 2020 17:16:31 +0200
Subject: [PATCH 120/716] add mention of FTP server storage in media storage
 intro

At the beginning of this doc, in "Specifying where to store the media" feature details, FTP server storage mention was missing
---
 docs/topics/media-pipeline.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 1f995ce14..c2255ea79 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -15,7 +15,7 @@ typically you'll either use the Files Pipeline or the Images Pipeline.
 Both pipelines implement these features:
 
 * Avoid re-downloading media that was downloaded recently
-* Specifying where to store the media (filesystem directory, Amazon S3 bucket,
+* Specifying where to store the media (filesystem directory, FTP server, Amazon S3 bucket,
   Google Cloud Storage bucket)
 
 The Images Pipeline has a few extra functions for processing images:

From 82ba7c8b529e004a62db4681f16a482ae4e769f1 Mon Sep 17 00:00:00 2001
From: drs-11 <siddharthadr11@gmail.com>
Date: Thu, 10 Sep 2020 20:56:39 +0530
Subject: [PATCH 121/716] created separate test for invalid no-proxy values

---
 tests/test_downloadermiddleware_httpproxy.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 81d6cc335..7c97bf32a 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -123,11 +123,7 @@ class TestHttpProxyMiddleware(TestCase):
 
     def test_no_proxy(self):
         os.environ['http_proxy'] = 'https://proxy.for.http:3128'
-        os.environ['no_proxy'] = '/var/run/docker.sock'
         mw = HttpProxyMiddleware()
-        # '/var/run/docker.sock' may be used by the user for
-        # no_proxy value but is not parseable and should be skipped
-        assert 'no' not in mw.proxies
 
         os.environ['no_proxy'] = '*'
         req = Request('http://noproxy.com')
@@ -149,3 +145,10 @@ class TestHttpProxyMiddleware(TestCase):
         req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'})
         assert mw.process_request(req, spider) is None
         self.assertEqual(req.meta, {'proxy': 'http://proxy.com'})
+
+    def test_no_proxy_invalid_values(self):
+        os.environ['no_proxy'] = '/var/run/docker.sock'
+        mw = HttpProxyMiddleware()
+        # '/var/run/docker.sock' may be used by the user for
+        # no_proxy value but is not parseable and should be skipped
+        assert 'no' not in mw.proxies

From 4d6359df2dfa3d561088faf097f6abe3f850a4d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 11 Sep 2020 13:51:05 +0200
Subject: [PATCH 122/716] Mark HTTP/2 as experimental

---
 docs/topics/asyncio.rst  | 11 ++++++-----
 docs/topics/commands.rst |  2 --
 docs/topics/settings.rst |  4 ++++
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index bfb430d52..c04044e8f 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -4,13 +4,14 @@ asyncio
 
 .. versionadded:: 2.0
 
-Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio
-reactor <install-asyncio>`, you may use :mod:`asyncio` and
+Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
+asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
 :mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
 
-.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
-             versions may introduce related changes without a deprecation
-             period or warning.
+.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet
+             recommended for production environments. Future Scrapy versions
+             may introduce related changes without a deprecation period or
+             warning.
 
 .. _install-asyncio:
 
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 7de5e8121..eef6b36ff 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -598,8 +598,6 @@ Example:
 Register commands via setup.py entry points
 -------------------------------------------
 
-.. note:: This is an experimental feature, use with caution.
-
 You can also add Scrapy commands from an external library by adding a
 ``scrapy.commands`` section in the entry points of the library ``setup.py``
 file.
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 8794e0259..218b23c87 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -678,6 +678,10 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
     Scrapy currently does not support HTTP/2 Cleartext (h2c) since none
     of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_).
 
+.. warning:: HTTP/2 support in Scrapy is experimental, and not yet recommended
+             for production environments. Future Scrapy versions may introduce
+             related changes without a deprecation period or warning.
+
 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
 
 .. setting:: DOWNLOAD_TIMEOUT

From 6e8d20a07a8c1a1f3ad7b3c3d74b7d37f2b47327 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Wed, 16 Sep 2020 04:57:07 -0300
Subject: [PATCH 123/716] HTTP/2: add some type hints (#4785)

---
 scrapy/core/downloader/handlers/http2.py | 24 +++++++++++++--------
 scrapy/core/http2/agent.py               | 27 ++++++++++++++----------
 scrapy/core/http2/protocol.py            | 15 ++++++++-----
 scrapy/core/http2/stream.py              |  7 +++---
 setup.cfg                                |  6 ------
 tests/test_downloader_handlers.py        | 19 +++++++++--------
 tests/test_http2_client_protocol.py      |  2 +-
 7 files changed, 56 insertions(+), 44 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 4be888bda..e97c31e90 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -1,8 +1,9 @@
 import warnings
 from time import time
-from typing import Optional
+from typing import Optional, Type, TypeVar
 from urllib.parse import urldefrag
 
+from twisted.internet.base import DelayedCall
 from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
 from twisted.web.client import URI
@@ -10,14 +11,18 @@ from twisted.web.client import URI
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
 from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
+from scrapy.crawler import Crawler
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.utils.python import to_bytes
 
 
+H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler")
+
+
 class H2DownloadHandler:
-    def __init__(self, settings: Settings, crawler=None):
+    def __init__(self, settings: Settings, crawler: Optional[Crawler] = None):
         self._crawler = crawler
 
         from twisted.internet import reactor
@@ -25,14 +30,14 @@ class H2DownloadHandler:
         self._context_factory = load_context_factory_from_settings(settings, crawler)
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass:
         return cls(crawler.settings, crawler)
 
     def download_request(self, request: Request, spider: Spider) -> Deferred:
         agent = ScrapyH2Agent(
             context_factory=self._context_factory,
             pool=self._pool,
-            crawler=self._crawler
+            crawler=self._crawler,
         )
         return agent.download_request(request, spider)
 
@@ -47,8 +52,9 @@ class ScrapyH2Agent:
     def __init__(
         self, context_factory,
         pool: H2ConnectionPool,
-        connect_timeout=10, bind_address: Optional[bytes] = None,
-        crawler=None
+        connect_timeout: int = 10,
+        bind_address: Optional[bytes] = None,
+        crawler: Optional[Crawler] = None,
     ) -> None:
         self._context_factory = context_factory
         self._connect_timeout = connect_timeout
@@ -80,7 +86,7 @@ class ScrapyH2Agent:
                 proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')),
                 connect_timeout=timeout,
                 bind_address=bind_address,
-                pool=self._pool
+                pool=self._pool,
             )
 
         return self._Agent(
@@ -88,7 +94,7 @@ class ScrapyH2Agent:
             context_factory=self._context_factory,
             connect_timeout=timeout,
             bind_address=bind_address,
-            pool=self._pool
+            pool=self._pool,
         )
 
     def download_request(self, request: Request, spider: Spider) -> Deferred:
@@ -110,7 +116,7 @@ class ScrapyH2Agent:
         return response
 
     @staticmethod
-    def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl) -> Response:
+    def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response:
         if timeout_cl.active():
             timeout_cl.cancel()
             return response
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index d950c6cfb..a142fa210 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -1,5 +1,5 @@
 from collections import deque
-from typing import Deque, Dict, List, Tuple, Optional
+from typing import Deque, Dict, List, Optional, Tuple
 
 from twisted.internet import defer
 from twisted.internet.base import ReactorBase
@@ -95,16 +95,18 @@ class H2ConnectionPool:
 
 class H2Agent:
     def __init__(
-        self, reactor: ReactorBase, pool: H2ConnectionPool,
-        context_factory=BrowserLikePolicyForHTTPS(),
-        connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
+        self,
+        reactor: ReactorBase,
+        pool: H2ConnectionPool,
+        context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),
+        connect_timeout: Optional[float] = None,
+        bind_address: Optional[bytes] = None,
     ) -> None:
         self._reactor = reactor
         self._pool = pool
         self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2'])
         self.endpoint_factory = _StandardEndpointFactory(
-            self._reactor, self._context_factory,
-            connect_timeout, bind_address
+            self._reactor, self._context_factory, connect_timeout, bind_address
         )
 
     def get_endpoint(self, uri: URI):
@@ -132,17 +134,20 @@ class H2Agent:
 
 class ScrapyProxyH2Agent(H2Agent):
     def __init__(
-        self, reactor: ReactorBase,
-        proxy_uri: URI, pool: H2ConnectionPool,
-        context_factory=BrowserLikePolicyForHTTPS(),
-        connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
+        self,
+        reactor: ReactorBase,
+        proxy_uri: URI,
+        pool: H2ConnectionPool,
+        context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),
+        connect_timeout: Optional[float] = None,
+        bind_address: Optional[bytes] = None,
     ) -> None:
         super(ScrapyProxyH2Agent, self).__init__(
             reactor=reactor,
             pool=pool,
             context_factory=context_factory,
             connect_timeout=connect_timeout,
-            bind_address=bind_address
+            bind_address=bind_address,
         )
         self._proxy_uri = proxy_uri
 
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index e32e2b6fe..9d499596c 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -29,6 +29,7 @@ from scrapy.http import Request
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -42,7 +43,11 @@ class InvalidNegotiatedProtocol(H2Error):
 
 
 class RemoteTerminatedConnection(H2Error):
-    def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], event: ConnectionTerminated):
+    def __init__(
+        self,
+        remote_ip_address: Optional[Union[IPv4Address, IPv6Address]],
+        event: ConnectionTerminated,
+    ) -> None:
         self.remote_ip_address = remote_ip_address
         self.terminate_event = event
 
@@ -51,7 +56,7 @@ class RemoteTerminatedConnection(H2Error):
 
 
 class MethodNotAllowed405(H2Error):
-    def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]):
+    def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None:
         self.remote_ip_address = remote_ip_address
 
     def __str__(self) -> str:
@@ -220,13 +225,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self.conn.initiate_connection()
         self._write_to_transport()
 
-    def _lose_connection_with_error(self, errors: List[BaseException]):
+    def _lose_connection_with_error(self, errors: List[BaseException]) -> None:
         """Helper function to lose the connection with the error sent as a
         reason"""
         self._conn_lost_errors += errors
         self.transport.loseConnection()
 
-    def handshakeCompleted(self):
+    def handshakeCompleted(self) -> None:
         """We close the connection with InvalidNegotiatedProtocol exception
         when the connection was not made via h2 protocol"""
         negotiated_protocol = self.transport.negotiatedProtocol
@@ -263,7 +268,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         finally:
             self._write_to_transport()
 
-    def timeoutConnection(self):
+    def timeoutConnection(self) -> None:
         """Called when the connection times out.
         We lose the connection with TimeoutError"""
 
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index ef90773b6..3ae2e8db8 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -19,6 +19,7 @@ from scrapy.responsetypes import responsetypes
 if TYPE_CHECKING:
     from scrapy.core.http2.protocol import H2ClientProtocol
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -27,7 +28,7 @@ class InactiveStreamClosed(ConnectionClosed):
     of the stream. This happens when a stream is waiting for other
     streams to close and connection is lost."""
 
-    def __init__(self, request: Request):
+    def __init__(self, request: Request) -> None:
         self.request = request
 
     def __str__(self) -> str:
@@ -139,7 +140,7 @@ class Stream:
             'headers': Headers({}),
         }
 
-        def _cancel(_):
+        def _cancel(_) -> None:
             # Close this stream as gracefully as possible
             # If the associated request is initiated we reset this stream
             # else we directly call close() method
@@ -360,7 +361,7 @@ class Stream:
         self,
         reason: StreamCloseReason,
         errors: Optional[List[BaseException]] = None,
-        from_protocol: bool = False
+        from_protocol: bool = False,
     ) -> None:
         """Based on the reason sent we will handle each case.
         """
diff --git a/setup.cfg b/setup.cfg
index 94c1e7b89..8101443e3 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -82,9 +82,6 @@ ignore_errors = True
 [mypy-tests.test_downloader_handlers]
 ignore_errors = True
 
-[mypy-tests.test_downloader_handlers_http2]
-ignore_errors = True
-
 [mypy-tests.test_engine]
 ignore_errors = True
 
@@ -94,9 +91,6 @@ ignore_errors = True
 [mypy-tests.test_http_request]
 ignore_errors = True
 
-[mypy-tests.test_http2_client_protocol]
-ignore_errors = True
-
 [mypy-tests.test_linkextractors]
 ignore_errors = True
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 20e31f7f7..5b4a2d270 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -2,6 +2,7 @@ import contextlib
 import os
 import shutil
 import tempfile
+from typing import Optional, Type
 from unittest import mock
 
 from testfixtures import LogCapture
@@ -206,7 +207,7 @@ class LargeChunkedFileResource(resource.Resource):
 
 class HttpTestCase(unittest.TestCase):
     scheme = 'http'
-    download_handler_cls = HTTPDownloadHandler
+    download_handler_cls: Type = HTTPDownloadHandler
 
     # only used for HTTPS tests
     keyfile = 'keys/localhost.key'
@@ -365,7 +366,7 @@ class HttpTestCase(unittest.TestCase):
 
 class Http10TestCase(HttpTestCase):
     """HTTP 1.0 test case"""
-    download_handler_cls = HTTP10DownloadHandler
+    download_handler_cls: Type = HTTP10DownloadHandler
 
 
 class Https10TestCase(Http10TestCase):
@@ -374,7 +375,7 @@ class Https10TestCase(Http10TestCase):
 
 class Http11TestCase(HttpTestCase):
     """HTTP 1.1 test case"""
-    download_handler_cls = HTTP11DownloadHandler
+    download_handler_cls: Type = HTTP11DownloadHandler
 
     def test_download_without_maxsize_limit(self):
         request = Request(self.getURL('file'))
@@ -561,7 +562,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
 
 class Https11CustomCiphers(unittest.TestCase):
     scheme = 'https'
-    download_handler_cls = HTTP11DownloadHandler
+    download_handler_cls: Type = HTTP11DownloadHandler
 
     keyfile = 'keys/localhost.key'
     certfile = 'keys/localhost.crt'
@@ -601,7 +602,7 @@ class Https11CustomCiphers(unittest.TestCase):
 
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
-    settings_dict = None
+    settings_dict: Optional[dict] = None
 
     def setUp(self):
         self.mockserver = MockServer()
@@ -668,7 +669,7 @@ class UriResource(resource.Resource):
 
 
 class HttpProxyTestCase(unittest.TestCase):
-    download_handler_cls = HTTPDownloadHandler
+    download_handler_cls: Type = HTTPDownloadHandler
     expected_http_proxy_request_body = b'http://example.com'
 
     def setUp(self):
@@ -721,14 +722,14 @@ class HttpProxyTestCase(unittest.TestCase):
 
 
 class Http10ProxyTestCase(HttpProxyTestCase):
-    download_handler_cls = HTTP10DownloadHandler
+    download_handler_cls: Type = HTTP10DownloadHandler
 
     def test_download_with_proxy_https_noconnect(self):
         raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler')
 
 
 class Http11ProxyTestCase(HttpProxyTestCase):
-    download_handler_cls = HTTP11DownloadHandler
+    download_handler_cls: Type = HTTP11DownloadHandler
 
     @defer.inlineCallbacks
     def test_download_with_proxy_https_timeout(self):
@@ -776,7 +777,7 @@ class S3AnonTestCase(unittest.TestCase):
 
 
 class S3TestCase(unittest.TestCase):
-    download_handler_cls = S3DownloadHandler
+    download_handler_cls: Type = S3DownloadHandler
 
     # test use same example keys than amazon developer guide
     # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 4926ada14..d9ab553f0 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -45,7 +45,7 @@ def make_html_body(val):
 
 class DummySpider(Spider):
     name = 'dummy'
-    start_urls = []
+    start_urls: list = []
 
     def parse(self, response):
         print(response)

From 5e997587d9b13344a0afa9bb4cf781829a66ce23 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Sun, 20 Sep 2020 18:06:46 +0500
Subject: [PATCH 124/716] Remove dead boto2 code, deprecate is_botocore()
 (#4776)

---
 scrapy/core/downloader/handlers/s3.py | 55 ++++-----------
 scrapy/extensions/feedexport.py       | 35 ++++------
 scrapy/pipelines/files.py             | 97 +++++++++------------------
 scrapy/utils/boto.py                  | 23 ++++++-
 scrapy/utils/test.py                  | 29 +++-----
 tests/test_feedexport.py              | 69 ++-----------------
 tests/test_pipeline_files.py          | 15 ++---
 7 files changed, 96 insertions(+), 227 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 0ef977893..1966570d4 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -2,41 +2,20 @@ from urllib.parse import unquote
 
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 from scrapy.exceptions import NotConfigured
-from scrapy.utils.boto import is_botocore
+from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import create_instance
 
 
-def _get_boto_connection():
-    from boto.s3.connection import S3Connection
-
-    class _v19_S3Connection(S3Connection):
-        """A dummy S3Connection wrapper that doesn't do any synchronous download"""
-        def _mexe(self, method, bucket, key, headers, *args, **kwargs):
-            return headers
-
-    class _v20_S3Connection(S3Connection):
-        """A dummy S3Connection wrapper that doesn't do any synchronous download"""
-        def _mexe(self, http_request, *args, **kwargs):
-            http_request.authorize(connection=self)
-            return http_request.headers
-
-    try:
-        import boto.auth  # noqa: F401
-    except ImportError:
-        _S3Connection = _v19_S3Connection
-    else:
-        _S3Connection = _v20_S3Connection
-
-    return _S3Connection
-
-
 class S3DownloadHandler:
 
     def __init__(self, settings, *,
                  crawler=None,
                  aws_access_key_id=None, aws_secret_access_key=None,
                  httpdownloadhandler=HTTPDownloadHandler, **kw):
+        if not is_botocore_available():
+            raise NotConfigured('missing botocore library')
+
         if not aws_access_key_id:
             aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
         if not aws_secret_access_key:
@@ -51,23 +30,15 @@ class S3DownloadHandler:
         self.anon = kw.get('anon')
 
         self._signer = None
-        if is_botocore():
-            import botocore.auth
-            import botocore.credentials
-            kw.pop('anon', None)
-            if kw:
-                raise TypeError(f'Unexpected keyword arguments: {kw}')
-            if not self.anon:
-                SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
-                self._signer = SignerCls(botocore.credentials.Credentials(
-                    aws_access_key_id, aws_secret_access_key))
-        else:
-            _S3Connection = _get_boto_connection()
-            try:
-                self.conn = _S3Connection(
-                    aws_access_key_id, aws_secret_access_key, **kw)
-            except Exception as ex:
-                raise NotConfigured(str(ex))
+        import botocore.auth
+        import botocore.credentials
+        kw.pop('anon', None)
+        if kw:
+            raise TypeError(f'Unexpected keyword arguments: {kw}')
+        if not self.anon:
+            SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
+            self._signer = SignerCls(botocore.credentials.Credentials(
+                aws_access_key_id, aws_secret_access_key))
 
         _http_handler = create_instance(
             objcls=httpdownloadhandler,
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 980825499..9f712285f 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -19,7 +19,7 @@ from zope.interface import implementer, Interface
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.utils.boto import is_botocore
+from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.conf import feed_complete_default_values_from_settings
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
@@ -120,22 +120,19 @@ class S3FeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,
                  feed_options=None):
+        if not is_botocore_available():
+            raise NotConfigured('missing botocore library')
         u = urlparse(uri)
         self.bucketname = u.hostname
         self.access_key = u.username or access_key
         self.secret_key = u.password or secret_key
-        self.is_botocore = is_botocore()
         self.keyname = u.path[1:]  # remove first "/"
         self.acl = acl
-        if self.is_botocore:
-            import botocore.session
-            session = botocore.session.get_session()
-            self.s3_client = session.create_client(
-                's3', aws_access_key_id=self.access_key,
-                aws_secret_access_key=self.secret_key)
-        else:
-            import boto
-            self.connect_s3 = boto.connect_s3
+        import botocore.session
+        session = botocore.session.get_session()
+        self.s3_client = session.create_client(
+            's3', aws_access_key_id=self.access_key,
+            aws_secret_access_key=self.secret_key)
         if feed_options and feed_options.get('overwrite', True) is False:
             logger.warning('S3 does not support appending to files. To '
                            'suppress this warning, remove the overwrite '
@@ -154,18 +151,10 @@ class S3FeedStorage(BlockingFeedStorage):
 
     def _store_in_thread(self, file):
         file.seek(0)
-        if self.is_botocore:
-            kwargs = {'ACL': self.acl} if self.acl else {}
-            self.s3_client.put_object(
-                Bucket=self.bucketname, Key=self.keyname, Body=file,
-                **kwargs)
-        else:
-            conn = self.connect_s3(self.access_key, self.secret_key)
-            bucket = conn.get_bucket(self.bucketname, validate=False)
-            key = bucket.new_key(self.keyname)
-            kwargs = {'policy': self.acl} if self.acl else {}
-            key.set_contents_from_file(file, **kwargs)
-            key.close()
+        kwargs = {'ACL': self.acl} if self.acl else {}
+        self.s3_client.put_object(
+            Bucket=self.bucketname, Key=self.keyname, Body=file,
+            **kwargs)
         file.close()
 
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 99a72aa70..13ecd4e6c 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -11,7 +11,6 @@ import os
 import time
 from collections import defaultdict
 from contextlib import suppress
-from email.utils import mktime_tz, parsedate_tz
 from ftplib import FTP
 from io import BytesIO
 from urllib.parse import urlparse
@@ -23,7 +22,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request
 from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
-from scrapy.utils.boto import is_botocore
+from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
@@ -91,86 +90,54 @@ class S3FilesStore:
     }
 
     def __init__(self, uri):
-        self.is_botocore = is_botocore()
-        if self.is_botocore:
-            import botocore.session
-            session = botocore.session.get_session()
-            self.s3_client = session.create_client(
-                's3',
-                aws_access_key_id=self.AWS_ACCESS_KEY_ID,
-                aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
-                endpoint_url=self.AWS_ENDPOINT_URL,
-                region_name=self.AWS_REGION_NAME,
-                use_ssl=self.AWS_USE_SSL,
-                verify=self.AWS_VERIFY
-            )
-        else:
-            from boto.s3.connection import S3Connection
-            self.S3Connection = S3Connection
+        if not is_botocore_available():
+            raise NotConfigured('missing botocore library')
+        import botocore.session
+        session = botocore.session.get_session()
+        self.s3_client = session.create_client(
+            's3',
+            aws_access_key_id=self.AWS_ACCESS_KEY_ID,
+            aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
+            endpoint_url=self.AWS_ENDPOINT_URL,
+            region_name=self.AWS_REGION_NAME,
+            use_ssl=self.AWS_USE_SSL,
+            verify=self.AWS_VERIFY
+        )
         if not uri.startswith("s3://"):
             raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
         self.bucket, self.prefix = uri[5:].split('/', 1)
 
     def stat_file(self, path, info):
         def _onsuccess(boto_key):
-            if self.is_botocore:
-                checksum = boto_key['ETag'].strip('"')
-                last_modified = boto_key['LastModified']
-                modified_stamp = time.mktime(last_modified.timetuple())
-            else:
-                checksum = boto_key.etag.strip('"')
-                last_modified = boto_key.last_modified
-                modified_tuple = parsedate_tz(last_modified)
-                modified_stamp = int(mktime_tz(modified_tuple))
+            checksum = boto_key['ETag'].strip('"')
+            last_modified = boto_key['LastModified']
+            modified_stamp = time.mktime(last_modified.timetuple())
             return {'checksum': checksum, 'last_modified': modified_stamp}
 
         return self._get_boto_key(path).addCallback(_onsuccess)
 
-    def _get_boto_bucket(self):
-        # disable ssl (is_secure=False) because of this python bug:
-        # https://bugs.python.org/issue5103
-        c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
-        return c.get_bucket(self.bucket, validate=False)
-
     def _get_boto_key(self, path):
         key_name = f'{self.prefix}{path}'
-        if self.is_botocore:
-            return threads.deferToThread(
-                self.s3_client.head_object,
-                Bucket=self.bucket,
-                Key=key_name)
-        else:
-            b = self._get_boto_bucket()
-            return threads.deferToThread(b.get_key, key_name)
+        return threads.deferToThread(
+            self.s3_client.head_object,
+            Bucket=self.bucket,
+            Key=key_name)
 
     def persist_file(self, path, buf, info, meta=None, headers=None):
         """Upload file to S3 storage"""
         key_name = f'{self.prefix}{path}'
         buf.seek(0)
-        if self.is_botocore:
-            extra = self._headers_to_botocore_kwargs(self.HEADERS)
-            if headers:
-                extra.update(self._headers_to_botocore_kwargs(headers))
-            return threads.deferToThread(
-                self.s3_client.put_object,
-                Bucket=self.bucket,
-                Key=key_name,
-                Body=buf,
-                Metadata={k: str(v) for k, v in (meta or {}).items()},
-                ACL=self.POLICY,
-                **extra)
-        else:
-            b = self._get_boto_bucket()
-            k = b.new_key(key_name)
-            if meta:
-                for metakey, metavalue in meta.items():
-                    k.set_metadata(metakey, str(metavalue))
-            h = self.HEADERS.copy()
-            if headers:
-                h.update(headers)
-            return threads.deferToThread(
-                k.set_contents_from_string, buf.getvalue(),
-                headers=h, policy=self.POLICY)
+        extra = self._headers_to_botocore_kwargs(self.HEADERS)
+        if headers:
+            extra.update(self._headers_to_botocore_kwargs(headers))
+        return threads.deferToThread(
+            self.s3_client.put_object,
+            Bucket=self.bucket,
+            Key=key_name,
+            Body=buf,
+            Metadata={k: str(v) for k, v in (meta or {}).items()},
+            ACL=self.POLICY,
+            **extra)
 
     def _headers_to_botocore_kwargs(self, headers):
         """ Convert headers to botocore keyword agruments.
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 12321caa5..3374c57c7 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,11 +1,32 @@
 """Boto/botocore helpers"""
+import warnings
 
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 
 
 def is_botocore():
+    """ Returns True if botocore is available, otherwise raises NotConfigured. Never returns False.
+
+    Previously, when boto was supported in addition to botocore, this returned False if boto was available
+    but botocore wasn't.
+    """
+    message = (
+        'is_botocore() is deprecated and always returns True or raises an Exception, '
+        'so it cannot be used for checking if boto is available instead of botocore. '
+        'You can use scrapy.utils.boto.is_botocore_available() to check if botocore '
+        'is available.'
+    )
+    warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
     try:
         import botocore  # noqa: F401
         return True
     except ImportError:
         raise NotConfigured('missing botocore library')
+
+
+def is_botocore_available():
+    try:
+        import botocore  # noqa: F401
+        return True
+    except ImportError:
+        return False
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index f54942ffb..94d0ae2d3 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -10,8 +10,7 @@ from unittest import mock
 from importlib import import_module
 from twisted.trial.unittest import SkipTest
 
-from scrapy.exceptions import NotConfigured
-from scrapy.utils.boto import is_botocore
+from scrapy.utils.boto import is_botocore_available
 
 
 def assert_aws_environ():
@@ -29,29 +28,19 @@ def assert_gcs_environ():
 
 
 def skip_if_no_boto():
-    try:
-        is_botocore()
-    except NotConfigured as e:
-        raise SkipTest(e)
+    if not is_botocore_available():
+        raise SkipTest('missing botocore library')
 
 
 def get_s3_content_and_delete(bucket, path, with_key=False):
     """ Get content from s3 key, and delete key afterwards.
     """
-    if is_botocore():
-        import botocore.session
-        session = botocore.session.get_session()
-        client = session.create_client('s3')
-        key = client.get_object(Bucket=bucket, Key=path)
-        content = key['Body'].read()
-        client.delete_object(Bucket=bucket, Key=path)
-    else:
-        import boto
-        # assuming boto=2.2.2
-        bucket = boto.connect_s3().get_bucket(bucket, validate=False)
-        key = bucket.get_key(path)
-        content = key.get_contents_as_string()
-        bucket.delete_key(path)
+    import botocore.session
+    session = botocore.session.get_session()
+    client = session.create_client('s3')
+    key = client.get_object(Bucket=bucket, Key=path)
+    content = key['Body'].read()
+    client.delete_object(Bucket=bucket, Key=path)
     return (content, key) if with_key else content
 
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 840e0f87b..33ac51712 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -45,6 +45,7 @@ from scrapy.utils.test import (
     get_s3_content_and_delete,
     get_crawler,
     mock_google_cloud_storage,
+    skip_if_no_boto,
 )
 
 from tests.mockserver import MockFTPServer, MockServer
@@ -227,10 +228,7 @@ class BlockingFeedStorageTest(unittest.TestCase):
 class S3FeedStorageTest(unittest.TestCase):
 
     def test_parse_credentials(self):
-        try:
-            import botocore  # noqa: F401
-        except ImportError:
-            raise unittest.SkipTest("S3FeedStorage requires botocore")
+        skip_if_no_boto()
         aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',
                            'AWS_SECRET_ACCESS_KEY': 'settings_secret'}
         crawler = get_crawler(settings_dict=aws_credentials)
@@ -324,11 +322,7 @@ class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store_botocore_without_acl(self):
-        try:
-            import botocore  # noqa: F401
-        except ImportError:
-            raise unittest.SkipTest('botocore is required')
-
+        skip_if_no_boto()
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
             'access_key',
@@ -344,11 +338,7 @@ class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store_botocore_with_acl(self):
-        try:
-            import botocore  # noqa: F401
-        except ImportError:
-            raise unittest.SkipTest('botocore is required')
-
+        skip_if_no_boto()
         storage = S3FeedStorage(
             's3://mybucket/export.csv',
             'access_key',
@@ -366,57 +356,6 @@ class S3FeedStorageTest(unittest.TestCase):
             'custom-acl'
         )
 
-    @defer.inlineCallbacks
-    def test_store_not_botocore_without_acl(self):
-        storage = S3FeedStorage(
-            's3://mybucket/export.csv',
-            'access_key',
-            'secret_key',
-        )
-        self.assertEqual(storage.access_key, 'access_key')
-        self.assertEqual(storage.secret_key, 'secret_key')
-        self.assertEqual(storage.acl, None)
-
-        storage.is_botocore = False
-        storage.connect_s3 = mock.MagicMock()
-        self.assertFalse(storage.is_botocore)
-
-        yield storage.store(BytesIO(b'test file'))
-
-        conn = storage.connect_s3(*storage.connect_s3.call_args)
-        bucket = conn.get_bucket(*conn.get_bucket.call_args)
-        key = bucket.new_key(*bucket.new_key.call_args)
-        self.assertNotIn(
-            dict(policy='custom-acl'),
-            key.set_contents_from_file.call_args
-        )
-
-    @defer.inlineCallbacks
-    def test_store_not_botocore_with_acl(self):
-        storage = S3FeedStorage(
-            's3://mybucket/export.csv',
-            'access_key',
-            'secret_key',
-            'custom-acl'
-        )
-        self.assertEqual(storage.access_key, 'access_key')
-        self.assertEqual(storage.secret_key, 'secret_key')
-        self.assertEqual(storage.acl, 'custom-acl')
-
-        storage.is_botocore = False
-        storage.connect_s3 = mock.MagicMock()
-        self.assertFalse(storage.is_botocore)
-
-        yield storage.store(BytesIO(b'test file'))
-
-        conn = storage.connect_s3(*storage.connect_s3.call_args)
-        bucket = conn.get_bucket(*conn.get_bucket.call_args)
-        key = bucket.new_key(*bucket.new_key.call_args)
-        self.assertIn(
-            dict(policy='custom-acl'),
-            key.set_contents_from_file.call_args
-        )
-
     def test_overwrite_default(self):
         with LogCapture() as log:
             S3FeedStorage(
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 1dd7031fe..d5b0bb3d8 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -22,7 +22,6 @@ from scrapy.pipelines.files import (
     S3FilesStore,
 )
 from scrapy.settings import Settings
-from scrapy.utils.boto import is_botocore
 from scrapy.utils.test import (
     assert_aws_environ,
     assert_gcs_environ,
@@ -437,16 +436,10 @@ class TestS3FilesStore(unittest.TestCase):
         content, key = get_s3_content_and_delete(
             u.hostname, u.path[1:], with_key=True)
         self.assertEqual(content, data)
-        if is_botocore():
-            self.assertEqual(key['Metadata'], {'foo': 'bar'})
-            self.assertEqual(
-                key['CacheControl'], S3FilesStore.HEADERS['Cache-Control'])
-            self.assertEqual(key['ContentType'], 'image/png')
-        else:
-            self.assertEqual(key.metadata, {'foo': 'bar'})
-            self.assertEqual(
-                key.cache_control, S3FilesStore.HEADERS['Cache-Control'])
-            self.assertEqual(key.content_type, 'image/png')
+        self.assertEqual(key['Metadata'], {'foo': 'bar'})
+        self.assertEqual(
+            key['CacheControl'], S3FilesStore.HEADERS['Cache-Control'])
+        self.assertEqual(key['ContentType'], 'image/png')
 
 
 class TestGCSFilesStore(unittest.TestCase):

From 70c82d33c00538228314dd6cef0253b70f8627e8 Mon Sep 17 00:00:00 2001
From: Georgiy Zatserklianyi <GeorgeA92@users.noreply.github.com>
Date: Sun, 20 Sep 2020 16:24:05 +0300
Subject: [PATCH 125/716] httpcompression stats added (#4797)

---
 scrapy/downloadermiddlewares/httpcompression.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 727c41466..b34f76f21 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -37,6 +37,8 @@ class HttpCompressionMiddleware:
             if content_encoding:
                 encoding = content_encoding.pop()
                 decoded_body = self._decode(response.body, encoding.lower())
+                spider.crawler.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
+                spider.crawler.stats.inc_value('httpcompression/response_count', spider=spider)
                 respcls = responsetypes.from_args(
                     headers=response.headers, url=response.url, body=decoded_body
                 )

From 7f1e74daa28caa481b4a484392a7dd2a18fc290e Mon Sep 17 00:00:00 2001
From: Mirwaisse Djanbaz <compsi75@gmail.com>
Date: Mon, 21 Sep 2020 14:38:16 +0200
Subject: [PATCH 126/716] =?UTF-8?q?dependencides=20=E2=86=92=20dependencie?=
 =?UTF-8?q?s=20(#4800)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/intro/install.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 8b4240bf6..fe7bc0a2a 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -211,7 +211,7 @@ PyPy
 We recommend using the latest PyPy version. The version tested is 5.9.0.
 For PyPy3, only Linux installation was tested.
 
-Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy.
+Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
 This means that these dependecies will be built during installation.
 On macOS, you are likely to face an issue with building Cryptography dependency,
 solution to this problem is described

From 3989f64baa39f7e42b0f798dec15cd250e0fba21 Mon Sep 17 00:00:00 2001
From: Mirwaisse Djanbaz <compsi75@gmail.com>
Date: Mon, 21 Sep 2020 14:40:00 +0200
Subject: [PATCH 127/716] =?UTF-8?q?dependecies=20=E2=86=92=20dependencies?=
 =?UTF-8?q?=20(#4801)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/intro/install.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index fe7bc0a2a..3bfd3bc3b 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -212,7 +212,7 @@ We recommend using the latest PyPy version. The version tested is 5.9.0.
 For PyPy3, only Linux installation was tested.
 
 Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
-This means that these dependecies will be built during installation.
+This means that these dependencies will be built during installation.
 On macOS, you are likely to face an issue with building Cryptography dependency,
 solution to this problem is described
 `here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,

From 008cf1c75ebe72a607149d7efd8f902f8763bc52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Sep 2020 20:45:21 +0200
Subject: [PATCH 128/716] Remove a test that has never been executed in Python
 3

---
 tests/test_downloader_handlers.py | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 0a374c161..3e8d7e6b9 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -868,29 +868,6 @@ class S3TestCase(unittest.TestCase):
         self.assertEqual(httpreq.headers['Authorization'],
                          b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
 
-    def test_request_signing5(self):
-        try:
-            import botocore  # noqa: F401
-        except ImportError:
-            pass
-        else:
-            raise unittest.SkipTest(
-                'botocore does not support overriding date with x-amz-date')
-        # deletes an object from the 'johnsmith' bucket using the
-        # path-style and Date alternative.
-        date = 'Tue, 27 Mar 2007 21:20:27 +0000'
-        req = Request(
-            's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={
-                'Date': date,
-                'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
-            })
-        with self._mocked_date(date):
-            httpreq = self.download_request(req, self.spider)
-        # botocore does not override Date with x-amz-date
-        self.assertEqual(
-            httpreq.headers['Authorization'],
-            b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
-
     def test_request_signing6(self):
         # uploads an object to a CNAME style virtual hosted bucket with metadata.
         date = 'Tue, 27 Mar 2007 21:06:08 +0000'

From 56f05fb16476b40f773edc9e93d41c8893a349c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Sep 2020 22:01:09 +0200
Subject: [PATCH 129/716] Use mocking for
 tests/test_feedexport.py::S3FeedStorageTest::test_store

---
 tests/test_feedexport.py | 50 +++++++++++++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 14 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 33ac51712..a8c0cf686 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -13,6 +13,7 @@ from logging import getLogger
 from pathlib import Path
 from string import ascii_letters, digits
 from unittest import mock
+from unittest.mock import call
 from urllib.parse import urljoin, urlparse, quote
 from urllib.request import pathname2url
 
@@ -254,21 +255,42 @@ class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store(self):
-        assert_aws_environ()
-        uri = os.environ.get('S3_TEST_FILE_URI')
-        if not uri:
-            raise unittest.SkipTest("No S3 URI available for testing")
-        access_key = os.environ.get('AWS_ACCESS_KEY_ID')
-        secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
-        storage = S3FeedStorage(uri, access_key, secret_key)
+        skip_if_no_boto()
+
+        settings = {
+            'AWS_ACCESS_KEY_ID': 'access_key',
+            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+        }
+        crawler = get_crawler(settings_dict=settings)
+        bucket = 'mybucket'
+        key = 'export.csv'
+        storage = S3FeedStorage.from_crawler(crawler, f's3://{bucket}/{key}')
         verifyObject(IFeedStorage, storage)
-        file = storage.open(scrapy.Spider("default"))
-        expected_content = b"content: \xe2\x98\x83"
-        file.write(expected_content)
-        yield storage.store(file)
-        u = urlparse(uri)
-        content = get_s3_content_and_delete(u.hostname, u.path[1:])
-        self.assertEqual(content, expected_content)
+
+        file = mock.MagicMock()
+        from botocore.stub import Stubber
+        with Stubber(storage.s3_client) as stub:
+            stub.add_response(
+                'put_object',
+                expected_params={
+                    'Body': file,
+                    'Bucket': bucket,
+                    'Key': key,
+                },
+                service_response={},
+            )
+
+            yield storage.store(file)
+
+            stub.assert_no_pending_responses()
+            self.assertEqual(
+                file.method_calls,
+                [
+                    call.seek(0),
+                    # The call to read does not happen with Stubber
+                    call.close(),
+                ]
+            )
 
     def test_init_without_acl(self):
         storage = S3FeedStorage(

From 17e135377a564152e82cd2ad177a6a2749c15832 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Sep 2020 22:54:39 +0200
Subject: [PATCH 130/716] Use mocking for
 tests/test_feedexport.py::BatchDeliveriesTest::test_s3_export

---
 tests/test_feedexport.py | 99 +++++++++++++++++++---------------------
 1 file changed, 48 insertions(+), 51 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index a8c0cf686..ac0b3ccaa 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -13,7 +13,6 @@ from logging import getLogger
 from pathlib import Path
 from string import ascii_letters, digits
 from unittest import mock
-from unittest.mock import call
 from urllib.parse import urljoin, urlparse, quote
 from urllib.request import pathname2url
 
@@ -42,7 +41,6 @@ from scrapy.extensions.feedexport import (
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import (
-    assert_aws_environ,
     get_s3_content_and_delete,
     get_crawler,
     mock_google_cloud_storage,
@@ -286,9 +284,9 @@ class S3FeedStorageTest(unittest.TestCase):
             self.assertEqual(
                 file.method_calls,
                 [
-                    call.seek(0),
+                    mock.call.seek(0),
                     # The call to read does not happen with Stubber
-                    call.close(),
+                    mock.call.close(),
                 ]
             )
 
@@ -1518,46 +1516,53 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_s3_export(self):
-        """
-        Test export of items into s3 bucket.
-        S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini
-        to perform this test:
-        [testenv]
-        setenv =
-            AWS_SECRET_ACCESS_KEY = ABCD
-            AWS_ACCESS_KEY_ID = EFGH
-            S3_TEST_BUCKET_NAME = IJKL
-        """
-        try:
-            import boto3
-        except ImportError:
-            raise unittest.SkipTest("S3FeedStorage requires boto3")
+        skip_if_no_boto()
 
-        assert_aws_environ()
-        s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME')
-        access_key = os.environ.get('AWS_ACCESS_KEY_ID')
-        secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
-        if not s3_test_bucket_name:
-            raise unittest.SkipTest("No S3 BUCKET available for testing")
-
-        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
-        filename = ''.join(chars)
-        prefix = f'tmp/{filename}'
-        s3_test_file_uri = f's3://{s3_test_bucket_name}/{prefix}/%(batch_time)s.json'
-        storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
-        settings = Settings({
-            'FEEDS': {
-                s3_test_file_uri: {
-                    'format': 'json',
-                },
-            },
-            'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
-        })
+        bucket = 'mybucket'
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
             self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
             self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
         ]
+
+        class CustomS3FeedStorage(S3FeedStorage):
+
+            stubs = []
+
+            def open(self, *args, **kwargs):
+                from botocore.stub import ANY, Stubber
+                stub = Stubber(self.s3_client)
+                stub.activate()
+                CustomS3FeedStorage.stubs.append(stub)
+                stub.add_response(
+                    'put_object',
+                    expected_params={
+                        'Body': ANY,
+                        'Bucket': bucket,
+                        'Key': ANY,
+                    },
+                    service_response={},
+                )
+                return super().open(*args, **kwargs)
+
+        key = 'export.csv'
+        uri = f's3://{bucket}/{key}/%(batch_time)s.json'
+        batch_item_count = 1
+        settings = {
+            'AWS_ACCESS_KEY_ID': 'access_key',
+            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+            'FEED_EXPORT_BATCH_ITEM_COUNT': batch_item_count,
+            'FEED_STORAGES': {
+                's3': CustomS3FeedStorage,
+            },
+            'FEEDS': {
+                uri: {
+                    'format': 'json',
+                },
+            },
+        }
+        crawler = get_crawler(settings_dict=settings)
+        storage = S3FeedStorage.from_crawler(crawler, uri)
         verifyObject(IFeedStorage, storage)
 
         class TestSpider(scrapy.Spider):
@@ -1567,22 +1572,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 for item in items:
                     yield item
 
-        s3 = boto3.resource('s3')
-        my_bucket = s3.Bucket(s3_test_bucket_name)
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
-
-        with MockServer() as s:
+        with MockServer() as server:
             runner = CrawlerRunner(Settings(settings))
-            TestSpider.start_urls = [s.url('/')]
+            TestSpider.start_urls = [server.url('/')]
             yield runner.crawl(TestSpider)
 
-        for file_uri in my_bucket.objects.filter(Prefix=prefix):
-            content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key)
-            if not content and not items:
-                break
-            content = json.loads(content.decode('utf-8'))
-            expected_batch, items = items[:batch_size], items[batch_size:]
-            self.assertEqual(expected_batch, content)
+        self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)+1)
+        for stub in CustomS3FeedStorage.stubs[:-1]:
+            stub.assert_no_pending_responses()
 
 
 class FeedExportInitTest(unittest.TestCase):

From 35726da434a9bc61dcb1ad5ef475c7d030023c87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Sep 2020 22:55:25 +0200
Subject: [PATCH 131/716] tests/test_feedexport.py: remove unused import

---
 tests/test_feedexport.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index ac0b3ccaa..18f7f8458 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -13,7 +13,7 @@ from logging import getLogger
 from pathlib import Path
 from string import ascii_letters, digits
 from unittest import mock
-from urllib.parse import urljoin, urlparse, quote
+from urllib.parse import urljoin, quote
 from urllib.request import pathname2url
 
 import lxml.etree

From c3b740f07814107e643bb0d073ea116049e37cba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Sep 2020 23:25:37 +0200
Subject: [PATCH 132/716] Use mocking for
 tests/test_pipeline_files.py::TestS3FilesStore::test_persist

---
 scrapy/utils/test.py         |   9 ----
 tests/test_pipeline_files.py | 100 +++++++++++++++++++++++++++--------
 2 files changed, 78 insertions(+), 31 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 94d0ae2d3..cf251442f 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -13,15 +13,6 @@ from twisted.trial.unittest import SkipTest
 from scrapy.utils.boto import is_botocore_available
 
 
-def assert_aws_environ():
-    """Asserts the current environment is suitable for running AWS testsi.
-    Raises SkipTest with the reason if it's not.
-    """
-    skip_if_no_boto()
-    if 'AWS_ACCESS_KEY_ID' not in os.environ:
-        raise SkipTest("AWS keys not found")
-
-
 def assert_gcs_environ():
     if 'GCS_PROJECT_ID' not in os.environ:
         raise SkipTest("GCS_PROJECT_ID not found")
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index d5b0bb3d8..1b14e3b1e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,6 +1,7 @@
 import os
 import random
 import time
+from datetime import datetime
 from io import BytesIO
 from shutil import rmtree
 from tempfile import mkdtemp
@@ -23,11 +24,11 @@ from scrapy.pipelines.files import (
 )
 from scrapy.settings import Settings
 from scrapy.utils.test import (
-    assert_aws_environ,
     assert_gcs_environ,
     get_ftp_content_and_delete,
     get_gcs_content_and_delete,
     get_s3_content_and_delete,
+    skip_if_no_boto,
 )
 
 
@@ -414,32 +415,87 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
 
 
 class TestS3FilesStore(unittest.TestCase):
+
     @defer.inlineCallbacks
     def test_persist(self):
-        assert_aws_environ()
-        uri = os.environ.get('S3_TEST_FILE_URI')
-        if not uri:
-            raise unittest.SkipTest("No S3 URI available for testing")
-        data = b"TestS3FilesStore: \xe2\x98\x83"
-        buf = BytesIO(data)
+        skip_if_no_boto()
+
+        bucket = 'mybucket'
+        key = 'export.csv'
+        uri = f's3://{bucket}/{key}'
+        buffer = mock.MagicMock()
         meta = {'foo': 'bar'}
         path = ''
+        content_type = 'image/png'
+
         store = S3FilesStore(uri)
-        yield store.persist_file(
-            path, buf, info=None, meta=meta,
-            headers={'Content-Type': 'image/png'})
-        s = yield store.stat_file(path, info=None)
-        self.assertIn('last_modified', s)
-        self.assertIn('checksum', s)
-        self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8')
-        u = urlparse(uri)
-        content, key = get_s3_content_and_delete(
-            u.hostname, u.path[1:], with_key=True)
-        self.assertEqual(content, data)
-        self.assertEqual(key['Metadata'], {'foo': 'bar'})
-        self.assertEqual(
-            key['CacheControl'], S3FilesStore.HEADERS['Cache-Control'])
-        self.assertEqual(key['ContentType'], 'image/png')
+        from botocore.stub import Stubber
+        with Stubber(store.s3_client) as stub:
+            stub.add_response(
+                'put_object',
+                expected_params={
+                    'ACL': S3FilesStore.POLICY,
+                    'Body': buffer,
+                    'Bucket': bucket,
+                    'CacheControl': S3FilesStore.HEADERS['Cache-Control'],
+                    'ContentType': content_type,
+                    'Key': key,
+                    'Metadata': meta,
+                },
+                service_response={},
+            )
+
+            yield store.persist_file(
+                path,
+                buffer,
+                info=None,
+                meta=meta,
+                headers={'Content-Type': content_type},
+            )
+
+            stub.assert_no_pending_responses()
+            self.assertEqual(
+                buffer.method_calls,
+                [
+                    mock.call.seek(0),
+                    # The call to read does not happen with Stubber
+                ]
+            )
+
+    @defer.inlineCallbacks
+    def test_stat(self):
+        skip_if_no_boto()
+
+        bucket = 'mybucket'
+        key = 'export.csv'
+        uri = f's3://{bucket}/{key}'
+        checksum = '3187896a9657a28163abb31667df64c8'
+
+        store = S3FilesStore(uri)
+        from botocore.stub import Stubber
+        with Stubber(store.s3_client) as stub:
+            stub.add_response(
+                'head_object',
+                expected_params={
+                    'Bucket': bucket,
+                    'Key': key,
+                },
+                service_response={
+                    'ETag': f'"{checksum}"',
+                    'LastModified': datetime(2019, 12, 1),
+                },
+            )
+
+            file_stats = yield store.stat_file('', info=None)
+            self.assertEqual(
+                file_stats,
+                {
+                    'checksum': checksum,
+                    'last_modified': 1575154800,
+                },
+            )
+
+            stub.assert_no_pending_responses()
 
 
 class TestGCSFilesStore(unittest.TestCase):

From 8f46e845190b7affb7a02c6842b465f0c5c3b76c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Sep 2020 23:28:16 +0200
Subject: [PATCH 133/716] Fix style issues

---
 scrapy/utils/test.py         | 12 ------------
 tests/test_feedexport.py     |  3 +--
 tests/test_pipeline_files.py |  1 -
 3 files changed, 1 insertion(+), 15 deletions(-)

diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index cf251442f..24c38283a 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -23,18 +23,6 @@ def skip_if_no_boto():
         raise SkipTest('missing botocore library')
 
 
-def get_s3_content_and_delete(bucket, path, with_key=False):
-    """ Get content from s3 key, and delete key afterwards.
-    """
-    import botocore.session
-    session = botocore.session.get_session()
-    client = session.create_client('s3')
-    key = client.get_object(Bucket=bucket, Key=path)
-    content = key['Body'].read()
-    client.delete_object(Bucket=bucket, Key=path)
-    return (content, key) if with_key else content
-
-
 def get_gcs_content_and_delete(bucket, path):
     from google.cloud import storage
     client = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 18f7f8458..3ed35bec5 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -41,7 +41,6 @@ from scrapy.extensions.feedexport import (
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import (
-    get_s3_content_and_delete,
     get_crawler,
     mock_google_cloud_storage,
     skip_if_no_boto,
@@ -1577,7 +1576,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             TestSpider.start_urls = [server.url('/')]
             yield runner.crawl(TestSpider)
 
-        self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)+1)
+        self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1)
         for stub in CustomS3FeedStorage.stubs[:-1]:
             stub.assert_no_pending_responses()
 
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 1b14e3b1e..e840298d9 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -27,7 +27,6 @@ from scrapy.utils.test import (
     assert_gcs_environ,
     get_ftp_content_and_delete,
     get_gcs_content_and_delete,
-    get_s3_content_and_delete,
     skip_if_no_boto,
 )
 

From 07c1d9c25b6f499fbe2cc0f0139d78cf6c1d204a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Sep 2020 23:32:55 +0200
Subject: [PATCH 134/716] Remove boto3 as a dependency for tests

---
 tox.ini | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 12e40295c..ea356c56a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,7 +12,6 @@ deps =
     -ctests/constraints.txt
     -rtests/requirements-py3.txt
     # Extras
-    boto3>=1.13.0
     botocore>=1.4.87
     Pillow>=4.0.0
 passenv =

From c22e810658b227095ea516ed61e51e6be41068ce Mon Sep 17 00:00:00 2001
From: GeorgeA92 <george.zatserklyany@gmail.com>
Date: Tue, 22 Sep 2020 07:47:37 +0300
Subject: [PATCH 135/716] httocompression tests added

---
 .../downloadermiddlewares/httpcompression.py  |  9 ++++---
 ...st_downloadermiddleware_httpcompression.py | 27 +++++++++++++++++--
 2 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index b34f76f21..ca80e9444 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -18,11 +18,14 @@ except ImportError:
 class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
+    def __init__(self, stats):
+        self.stats = stats
+
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('COMPRESSION_ENABLED'):
             raise NotConfigured
-        return cls()
+        return cls(crawler.stats)
 
     def process_request(self, request, spider):
         request.headers.setdefault('Accept-Encoding',
@@ -37,8 +40,8 @@ class HttpCompressionMiddleware:
             if content_encoding:
                 encoding = content_encoding.pop()
                 decoded_body = self._decode(response.body, encoding.lower())
-                spider.crawler.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
-                spider.crawler.stats.inc_value('httpcompression/response_count', spider=spider)
+                self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
+                self.stats.inc_value('httpcompression/response_count', spider=spider)
                 respcls = responsetypes.from_args(
                     headers=response.headers, url=response.url, body=decoded_body
                 )
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index a806f55ce..998749c2e 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -8,6 +8,7 @@ from scrapy.http import Response, Request, HtmlResponse
 from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.gz import gunzip
+from scrapy.utils.test import get_crawler
 from tests import tests_datadir
 from w3lib.encoding import resolve_encoding
 
@@ -26,8 +27,10 @@ FORMAT = {
 class HttpCompressionTest(TestCase):
 
     def setUp(self):
-        self.spider = Spider('foo')
-        self.mw = HttpCompressionMiddleware()
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('scrapytest.org')
+        self.mw = HttpCompressionMiddleware(self.crawler.stats)
+        self.crawler.stats.open_spider(self.spider)
 
     def _getresponse(self, coding):
         if coding not in FORMAT:
@@ -50,6 +53,13 @@ class HttpCompressionTest(TestCase):
         response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})
         return response
 
+    def assertStatsEqual(self, key, value):
+        self.assertEqual(
+            self.crawler.stats.get_value(key, spider=self.spider),
+            value,
+            str(self.crawler.stats.get_stats(self.spider))
+        )
+
     def test_process_request(self):
         request = Request('http://scrapytest.org')
         assert 'Accept-Encoding' not in request.headers
@@ -66,6 +76,7 @@ class HttpCompressionTest(TestCase):
         assert newresponse is not response
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
+        self.assertStatsEqual('httpcompression/response_count', 1)
 
     def test_process_response_br(self):
         try:
@@ -79,6 +90,7 @@ class HttpCompressionTest(TestCase):
         assert newresponse is not response
         assert newresponse.body.startswith(b"<!DOCTYPE")
         assert 'Content-Encoding' not in newresponse.headers
+        self.assertStatsEqual('httpcompression/response_count', 1)
 
     def test_process_response_rawdeflate(self):
         response = self._getresponse('rawdeflate')
@@ -89,6 +101,7 @@ class HttpCompressionTest(TestCase):
         assert newresponse is not response
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
+        self.assertStatsEqual('httpcompression/response_count', 1)
 
     def test_process_response_zlibdelate(self):
         response = self._getresponse('zlibdeflate')
@@ -99,6 +112,7 @@ class HttpCompressionTest(TestCase):
         assert newresponse is not response
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
+        self.assertStatsEqual('httpcompression/response_count', 1)
 
     def test_process_response_plain(self):
         response = Response('http://scrapytest.org', body=b'<!DOCTYPE...')
@@ -108,6 +122,7 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is response
         assert newresponse.body.startswith(b'<!DOCTYPE')
+        self.assertStatsEqual('httpcompression/response_count', None)
 
     def test_multipleencodings(self):
         response = self._getresponse('gzip')
@@ -135,6 +150,7 @@ class HttpCompressionTest(TestCase):
         assert isinstance(newresponse, HtmlResponse)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
+        self.assertStatsEqual('httpcompression/response_count', 1)
 
     def test_process_response_force_recalculate_encoding(self):
         headers = {
@@ -154,6 +170,7 @@ class HttpCompressionTest(TestCase):
         assert isinstance(newresponse, HtmlResponse)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
+        self.assertStatsEqual('httpcompression/response_count', 1)
 
     def test_process_response_no_content_type_header(self):
         headers = {
@@ -169,6 +186,7 @@ class HttpCompressionTest(TestCase):
         assert isinstance(newresponse, respcls)
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
+        self.assertStatsEqual('httpcompression/response_count', 1)
 
     def test_process_response_gzipped_contenttype(self):
         response = self._getresponse('gzip')
@@ -179,6 +197,7 @@ class HttpCompressionTest(TestCase):
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
         self.assertNotIn('Content-Encoding', newresponse.headers)
+        self.assertStatsEqual('httpcompression/response_count', 1)
 
     def test_process_response_gzip_app_octetstream_contenttype(self):
         response = self._getresponse('gzip')
@@ -189,6 +208,7 @@ class HttpCompressionTest(TestCase):
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
         self.assertNotIn('Content-Encoding', newresponse.headers)
+        self.assertStatsEqual('httpcompression/response_count', 1)
 
     def test_process_response_gzip_binary_octetstream_contenttype(self):
         response = self._getresponse('x-gzip')
@@ -199,6 +219,7 @@ class HttpCompressionTest(TestCase):
         self.assertIsNot(newresponse, response)
         self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
         self.assertNotIn('Content-Encoding', newresponse.headers)
+        self.assertStatsEqual('httpcompression/response_count', 1)
 
     def test_process_response_gzipped_gzip_file(self):
         """Test that a gzip Content-Encoded .gz file is gunzipped
@@ -241,6 +262,7 @@ class HttpCompressionTest(TestCase):
 
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertEqual(gunzip(newresponse.body), plainbody)
+        self.assertStatsEqual('httpcompression/response_count', 1)
 
     def test_process_response_head_request_no_decode_required(self):
         response = self._getresponse('gzip')
@@ -251,3 +273,4 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIs(newresponse, response)
         self.assertEqual(response.body, b'')
+        self.assertStatsEqual('httpcompression/response_count', None)

From 6ef7c44061f74281e7a977c9b1de4892cf84caf2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 22 Sep 2020 12:45:21 +0200
Subject: [PATCH 136/716] Fix timezone test issue

---
 tests/test_pipeline_files.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index e840298d9..4e1b90787 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -469,6 +469,7 @@ class TestS3FilesStore(unittest.TestCase):
         key = 'export.csv'
         uri = f's3://{bucket}/{key}'
         checksum = '3187896a9657a28163abb31667df64c8'
+        last_modified = datetime(2019, 12, 1)
 
         store = S3FilesStore(uri)
         from botocore.stub import Stubber
@@ -481,7 +482,7 @@ class TestS3FilesStore(unittest.TestCase):
                 },
                 service_response={
                     'ETag': f'"{checksum}"',
-                    'LastModified': datetime(2019, 12, 1),
+                    'LastModified': last_modified,
                 },
             )
 
@@ -490,7 +491,7 @@ class TestS3FilesStore(unittest.TestCase):
                 file_stats,
                 {
                     'checksum': checksum,
-                    'last_modified': 1575154800,
+                    'last_modified': last_modified.timestamp(),
                 },
             )
 

From eff96038c7a488860c1c58c2c7e37f888264c3dc Mon Sep 17 00:00:00 2001
From: madeny <7504281+madeny@users.noreply.github.com>
Date: Sat, 26 Sep 2020 22:50:38 +0200
Subject: [PATCH 137/716] Correct some typos

This won't be an issue if **your** spider doesn't rely on cookies.
---
 docs/topics/jobs.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 58601824a..d855d0133 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -65,7 +65,7 @@ Cookies expiration
 ------------------
 
 Cookies may expire. So, if you don't resume your spider quickly the requests
-scheduled may no longer work. This won't be an issue if you spider doesn't rely
+scheduled may no longer work. This won't be an issue if your spider doesn't rely
 on cookies.
 
 

From 9186e5a686e04703baf78974824917cf7a5121ec Mon Sep 17 00:00:00 2001
From: dswij <dharmasw@outlook.com>
Date: Tue, 29 Sep 2020 00:27:39 +0700
Subject: [PATCH 138/716] add pip 20.2 test for tox

---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index 12e40295c..f73eaef36 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,6 +23,8 @@ passenv =
     GCS_PROJECT_ID
 commands =
     py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
+install_command =
+    pip install --use-feature=2020-resolver {opts} {packages} 
 
 [testenv:typing]
 basepython = python3

From 894b509d7a4262cfc7548d78e9ff4831c6551c34 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 29 Sep 2020 23:37:28 -0300
Subject: [PATCH 139/716] Crawl rule: remove deprecated code

Remove the compatibility layer that handles 'process_request'
methods that do not receive a 'response' parameter
---
 scrapy/spiders/crawl.py | 34 ++++++++++---------------------
 tests/test_spider.py    | 45 +++++++++++++++++------------------------
 2 files changed, 30 insertions(+), 49 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index bc4551a54..1dcf2e6ab 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -6,14 +6,11 @@ See documentation in docs/topics/spiders.rst
 """
 
 import copy
-import warnings
 from typing import Sequence
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
-from scrapy.utils.python import get_func_args
 from scrapy.utils.spider import iterate_spider_output
 
 
@@ -37,15 +34,22 @@ _default_link_extractor = LinkExtractor()
 
 class Rule:
 
-    def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None,
-                 process_links=None, process_request=None, errback=None):
+    def __init__(
+        self,
+        link_extractor=None,
+        callback=None,
+        cb_kwargs=None,
+        follow=None,
+        process_links=None,
+        process_request=None,
+        errback=None,
+    ):
         self.link_extractor = link_extractor or _default_link_extractor
         self.callback = callback
         self.errback = errback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links or _identity
         self.process_request = process_request or _identity_process_request
-        self.process_request_argcount = None
         self.follow = follow if follow is not None else not callback
 
     def _compile(self, spider):
@@ -53,22 +57,6 @@ class Rule:
         self.errback = _get_method(self.errback, spider)
         self.process_links = _get_method(self.process_links, spider)
         self.process_request = _get_method(self.process_request, spider)
-        self.process_request_argcount = len(get_func_args(self.process_request))
-        if self.process_request_argcount == 1:
-            warnings.warn(
-                "Rule.process_request should accept two arguments "
-                "(request, response), accepting only one is deprecated",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
-
-    def _process_request(self, request, response):
-        """
-        Wrapper around the request processing function to maintain backward
-        compatibility with functions that do not take a Response object
-        """
-        args = [request] if self.process_request_argcount == 1 else [request, response]
-        return self.process_request(*args)
 
 
 class CrawlSpider(Spider):
@@ -111,7 +99,7 @@ class CrawlSpider(Spider):
             for link in rule.process_links(links):
                 seen.add(link)
                 request = self._build_request(rule_index, link)
-                yield rule._process_request(request, response)
+                yield rule.process_request(request, response)
 
     def _callback(self, response):
         rule = self._rules[response.meta['rule']]
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 78157a9b9..d23543f6a 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -1,8 +1,8 @@
 import gzip
 import inspect
-from unittest import mock
 import warnings
 from io import BytesIO
+from unittest import mock
 
 from testfixtures import LogCapture
 from twisted.trial import unittest
@@ -20,7 +20,6 @@ from scrapy.spiders import (
     XMLFeedSpider,
 )
 from scrapy.linkextractors import LinkExtractor
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.test import get_crawler
 
 
@@ -280,7 +279,7 @@ class CrawlSpiderTest(SpiderTest):
 
         response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
 
-        def process_request_change_domain(request):
+        def process_request_change_domain(request, response):
             return request.replace(url=request.url.replace('.org', '.com'))
 
         class _CrawlSpider(self.spider_class):
@@ -290,17 +289,14 @@ class CrawlSpiderTest(SpiderTest):
                 Rule(LinkExtractor(), process_request=process_request_change_domain),
             )
 
-        with warnings.catch_warnings(record=True) as cw:
-            spider = _CrawlSpider()
-            output = list(spider._requests_to_follow(response))
-            self.assertEqual(len(output), 3)
-            self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-            self.assertEqual([r.url for r in output],
-                             ['http://example.com/somepage/item/12.html',
-                              'http://example.com/about.html',
-                              'http://example.com/nofollow.html'])
-            self.assertEqual(len(cw), 1)
-            self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
+        spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
+                         ['http://example.com/somepage/item/12.html',
+                          'http://example.com/about.html',
+                          'http://example.com/nofollow.html'])
 
     def test_process_request_with_response(self):
 
@@ -339,20 +335,17 @@ class CrawlSpiderTest(SpiderTest):
                 Rule(LinkExtractor(), process_request='process_request_upper'),
             )
 
-            def process_request_upper(self, request):
+            def process_request_upper(self, request, response):
                 return request.replace(url=request.url.upper())
 
-        with warnings.catch_warnings(record=True) as cw:
-            spider = _CrawlSpider()
-            output = list(spider._requests_to_follow(response))
-            self.assertEqual(len(output), 3)
-            self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-            self.assertEqual([r.url for r in output],
-                             ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
-                              'http://EXAMPLE.ORG/ABOUT.HTML',
-                              'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
-            self.assertEqual(len(cw), 1)
-            self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
+        spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
+                         ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
+                          'http://EXAMPLE.ORG/ABOUT.HTML',
+                          'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
 
     def test_process_request_instance_method_with_response(self):
 

From 774ebe8796d204b38b332ee71a6ffa678c9b32b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 30 Sep 2020 14:17:30 +0200
Subject: [PATCH 140/716] Mention contributors in the README

---
 README.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/README.rst b/README.rst
index a8f2ba52b..19faa9a87 100644
--- a/README.rst
+++ b/README.rst
@@ -34,9 +34,16 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
 crawl websites and extract structured data from their pages. It can be used for
 a wide range of purposes, from data mining to monitoring and automated testing.
 
+Scrapy has contributions from `many users`_ (thanks everyone!) and is sponsored
+by `Scrapinghub Ltd`_.
+
+.. _many users: https://github.com/scrapy/scrapy/graphs/contributors
+.. _Scrapinghub Ltd: https://www.scrapinghub.com/
+
 Check the Scrapy homepage at https://scrapy.org for more information,
 including a list of features.
 
+
 Requirements
 ============
 

From 9661a8dcfc79249e8e3e117eb70682ebc8e57a92 Mon Sep 17 00:00:00 2001
From: Sashreek Shankar <45600974+sashreek1@users.noreply.github.com>
Date: Thu, 1 Oct 2020 06:46:12 +0530
Subject: [PATCH 141/716] removed datatype specification for *args & ** kwargs

---
 scrapy/crawler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 4c6b0e496..578016536 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -180,9 +180,9 @@ class CrawlerRunner:
         :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
             :class:`~scrapy.spiders.Spider` subclass or string
 
-        :param list args: arguments to initialize the spider
+        :param args: arguments to initialize the spider
 
-        :param dict kwargs: keyword arguments to initialize the spider
+        :param kwargs: keyword arguments to initialize the spider
         """
         if isinstance(crawler_or_spidercls, Spider):
             raise ValueError(

From f4629fe2cc8469e540b68a48a00fdcbb5ecd976a Mon Sep 17 00:00:00 2001
From: dswij <dharmasw@outlook.com>
Date: Thu, 1 Oct 2020 14:58:14 +0700
Subject: [PATCH 142/716] Update travis-pip and tox deps conflict for pip20.2

---
 .travis.yml | 1 +
 tox.ini     | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index b883c5b78..de5bbe9ab 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -49,6 +49,7 @@ install:
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
       fi
+  - python -m pip install --upgrade pip #force travis to use newest version of pip
   - pip install -U tox twine wheel codecov
 
 script: tox
diff --git a/tox.ini b/tox.ini
index f73eaef36..9a41fc240 100644
--- a/tox.ini
+++ b/tox.ini
@@ -64,7 +64,8 @@ commands =
 [pinned]
 deps =
     -ctests/constraints.txt
-    cryptography==2.0
+    #using cryptography-2.1.4 for test jobs to solve dependencies conflict on pip20.2>=
+    cryptography==2.1.4
     cssselect==0.9.1
     itemadapter==0.1.0
     parsel==1.5.0

From 66201737a0d245a2568f9b17541d38d6134fa03c Mon Sep 17 00:00:00 2001
From: dswij <dharmasw@outlook.com>
Date: Thu, 1 Oct 2020 15:47:19 +0700
Subject: [PATCH 143/716] fix travis and deps

---
 .travis.yml | 2 +-
 tox.ini     | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index de5bbe9ab..d53695e01 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -48,8 +48,8 @@ install:
         tar -jxf ${PYPY_VERSION}.tar.bz2
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
+        python -m pip install -U pip
       fi
-  - python -m pip install --upgrade pip #force travis to use newest version of pip
   - pip install -U tox twine wheel codecov
 
 script: tox
diff --git a/tox.ini b/tox.ini
index 9a41fc240..abbc0752e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -64,14 +64,15 @@ commands =
 [pinned]
 deps =
     -ctests/constraints.txt
-    #using cryptography-2.1.4 for test jobs to solve dependencies conflict on pip20.2>=
+    #using cryptography-2.1.4 to solve dependencies conflict on pip20.2>=
     cryptography==2.1.4
     cssselect==0.9.1
     itemadapter==0.1.0
     parsel==1.5.0
     Protego==0.1.15
     PyDispatcher==2.0.5
-    pyOpenSSL==16.2.0
+    #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>=
+    pyOpenSSL==18.0.0
     queuelib==1.4.2
     service_identity==16.0.0
     Twisted==17.9.0

From 0ea6ff11360963b7a26f11087cb8f62113070dd5 Mon Sep 17 00:00:00 2001
From: dswij <dharmasw@outlook.com>
Date: Thu, 1 Oct 2020 16:14:36 +0700
Subject: [PATCH 144/716] travis pip version and deps

---
 .travis.yml | 2 +-
 tox.ini     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index d53695e01..1ad5c4e9b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -48,8 +48,8 @@ install:
         tar -jxf ${PYPY_VERSION}.tar.bz2
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-        python -m pip install -U pip
       fi
+  - pip install --upgrade pip
   - pip install -U tox twine wheel codecov
 
 script: tox
diff --git a/tox.ini b/tox.ini
index abbc0752e..e923aeb85 100644
--- a/tox.ini
+++ b/tox.ini
@@ -64,8 +64,8 @@ commands =
 [pinned]
 deps =
     -ctests/constraints.txt
-    #using cryptography-2.1.4 to solve dependencies conflict on pip20.2>=
-    cryptography==2.1.4
+    #using cryptography-3.1 to solve dependencies conflict on pip20.2>=
+    cryptography==3.1
     cssselect==0.9.1
     itemadapter==0.1.0
     parsel==1.5.0

From f7201b1427749cc364c6f76100f01938d581340c Mon Sep 17 00:00:00 2001
From: dswij <dharmasw@outlook.com>
Date: Thu, 1 Oct 2020 19:59:23 +0700
Subject: [PATCH 145/716] travis and deps

---
 .travis.yml | 3 ++-
 tox.ini     | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 1ad5c4e9b..4c4bc0ef7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,6 +40,8 @@ matrix:
     - env: TOXENV=asyncio
       python: 3.8
       dist: bionic
+before_install:
+  - pip install -U pip
 install:
   - |
       if [[ ! -z "$PYPY_VERSION" ]]; then
@@ -49,7 +51,6 @@ install:
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
       fi
-  - pip install --upgrade pip
   - pip install -U tox twine wheel codecov
 
 script: tox
diff --git a/tox.ini b/tox.ini
index e923aeb85..e1e46921e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -64,8 +64,8 @@ commands =
 [pinned]
 deps =
     -ctests/constraints.txt
-    #using cryptography-3.1 to solve dependencies conflict on pip20.2>=
-    cryptography==3.1
+    #using cryptography-2.3.1 to solve dependencies conflict on pip20.2>=
+    cryptography==2.3.1
     cssselect==0.9.1
     itemadapter==0.1.0
     parsel==1.5.0

From 392b489a65b62244fa6cc1e1406c8e2e2d50d966 Mon Sep 17 00:00:00 2001
From: dswij <dharmasw@outlook.com>
Date: Thu, 1 Oct 2020 20:32:49 +0700
Subject: [PATCH 146/716] travis

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 4c4bc0ef7..da37c85c6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,6 +42,7 @@ matrix:
       dist: bionic
 before_install:
   - pip install -U pip
+  - pip --version
 install:
   - |
       if [[ ! -z "$PYPY_VERSION" ]]; then

From c83a16898f35c99b0383736e24293edbeaa5d46f Mon Sep 17 00:00:00 2001
From: dswij <dharmasw@outlook.com>
Date: Thu, 1 Oct 2020 20:59:05 +0700
Subject: [PATCH 147/716] try removing cache in travis to install pip

---
 .travis.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index da37c85c6..65fa3fa45 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -63,9 +63,6 @@ notifications:
     skip_join: true
     channels:
     - irc.freenode.org#scrapy
-cache:
-  directories:
-    - $HOME/.cache/pip
 deploy:
   provider: pypi
   distributions: "sdist bdist_wheel"

From 744f352d09a9ec519c84ec5243e62eef78c66f08 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 1 Oct 2020 14:52:23 -0300
Subject: [PATCH 148/716] Do not process cookies from headers

---
 scrapy/downloadermiddlewares/cookies.py    | 41 ++++++----------------
 tests/test_downloadermiddleware_cookies.py |  5 +++
 2 files changed, 15 insertions(+), 31 deletions(-)

diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 87f8152a4..d95ed3d38 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -97,35 +97,14 @@ class CookiesMiddleware:
 
     def _get_request_cookies(self, jar, request):
         """
-        Extract cookies from a Request. Values from the `Request.cookies` attribute
-        take precedence over values from the `Cookie` request header.
+        Extract cookies from the Request.cookies attribute
         """
-        def get_cookies_from_header(jar, request):
-            cookie_header = request.headers.get("Cookie")
-            if not cookie_header:
-                return []
-            cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";"))
-            cookie_list_unicode = []
-            for cookie_bytes in cookie_gen_bytes:
-                try:
-                    cookie_unicode = cookie_bytes.decode("utf8")
-                except UnicodeDecodeError:
-                    logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
-                                   request, cookie_bytes)
-                    cookie_unicode = cookie_bytes.decode("latin1", errors="replace")
-                cookie_list_unicode.append(cookie_unicode)
-            response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode})
-            return jar.make_cookies(response, request)
-
-        def get_cookies_from_attribute(jar, request):
-            if not request.cookies:
-                return []
-            elif isinstance(request.cookies, dict):
-                cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
-            else:
-                cookies = request.cookies
-            formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
-            response = Response(request.url, headers={"Set-Cookie": formatted})
-            return jar.make_cookies(response, request)
-
-        return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request)
+        if not request.cookies:
+            return []
+        elif isinstance(request.cookies, dict):
+            cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
+        else:
+            cookies = request.cookies
+        formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
+        response = Response(request.url, headers={"Set-Cookie": formatted})
+        return jar.make_cookies(response, request)
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index a3de307ee..aff8542e9 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -2,6 +2,8 @@ import logging
 from testfixtures import LogCapture
 from unittest import TestCase
 
+import pytest
+
 from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
 from scrapy.exceptions import NotConfigured
@@ -243,6 +245,7 @@ class CookiesMiddlewareTest(TestCase):
         self.assertIn('Cookie', request.headers)
         self.assertEqual(b'currencyCookie=USD', request.headers['Cookie'])
 
+    @pytest.mark.xfail(reason="Cookie header is not currently being processed")
     def test_keep_cookie_from_default_request_headers_middleware(self):
         DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty')
         mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())
@@ -257,6 +260,7 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(req2, self.spider) is None
         self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty')
 
+    @pytest.mark.xfail(reason="Cookie header is not currently being processed")
     def test_keep_cookie_header(self):
         # keep only cookies from 'Cookie' request header
         req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'})
@@ -291,6 +295,7 @@ class CookiesMiddlewareTest(TestCase):
         assert self.mw.process_request(req3, self.spider) is None
         self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
 
+    @pytest.mark.xfail(reason="Cookie header is not currently being processed")
     def test_request_headers_cookie_encoding(self):
         # 1) UTF8-encoded bytes
         req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')})

From cc81f9ed06399bd9c4e730a76f54032d9a7e9106 Mon Sep 17 00:00:00 2001
From: dswij <dharmasw@outlook.com>
Date: Fri, 2 Oct 2020 00:56:59 +0700
Subject: [PATCH 149/716] add download setting for tox

---
 .travis.yml | 4 +---
 tox.ini     | 1 +
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 65fa3fa45..5351abf87 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,9 +40,6 @@ matrix:
     - env: TOXENV=asyncio
       python: 3.8
       dist: bionic
-before_install:
-  - pip install -U pip
-  - pip --version
 install:
   - |
       if [[ ! -z "$PYPY_VERSION" ]]; then
@@ -63,6 +60,7 @@ notifications:
     skip_join: true
     channels:
     - irc.freenode.org#scrapy
+
 deploy:
   provider: pypi
   distributions: "sdist bdist_wheel"
diff --git a/tox.ini b/tox.ini
index e1e46921e..f51be432d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,6 +21,7 @@ passenv =
     AWS_SECRET_ACCESS_KEY
     GCS_TEST_FILE_URI
     GCS_PROJECT_ID
+download = true #allow tox virtualenv to upgarde pip/wheel/setuptools
 commands =
     py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
 install_command =

From 95b2e94496f86d1a225d573adc2af2facce35401 Mon Sep 17 00:00:00 2001
From: dswij <dharmasw@outlook.com>
Date: Fri, 2 Oct 2020 01:05:45 +0700
Subject: [PATCH 150/716] fix comment error on tox and re-add cache for travis

---
 .travis.yml | 4 +++-
 tox.ini     | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 5351abf87..b883c5b78 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -60,7 +60,9 @@ notifications:
     skip_join: true
     channels:
     - irc.freenode.org#scrapy
-
+cache:
+  directories:
+    - $HOME/.cache/pip
 deploy:
   provider: pypi
   distributions: "sdist bdist_wheel"
diff --git a/tox.ini b/tox.ini
index f51be432d..8cc522378 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,7 +21,8 @@ passenv =
     AWS_SECRET_ACCESS_KEY
     GCS_TEST_FILE_URI
     GCS_PROJECT_ID
-download = true #allow tox virtualenv to upgarde pip/wheel/setuptools
+#allow tox virtualenv to upgrade pip/wheel/setuptools
+download = true 
 commands =
     py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
 install_command =

From 44f0fde9057256dc16f24f22cfebc61626a94450 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Thu, 1 Oct 2020 23:21:09 +0500
Subject: [PATCH 151/716] Simplify TLS logging for the modern pyOpenSSL.
 (#4822)

---
 scrapy/core/downloader/tls.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index d9f3750d5..b5c6cc895 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -55,18 +55,11 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
             set_tlsext_host_name(connection, self._hostnameBytes)
         elif where & SSL.SSL_CB_HANDSHAKE_DONE:
             if self.verbose_logging:
-                if hasattr(connection, 'get_cipher_name'):  # requires pyOPenSSL 0.15
-                    if hasattr(connection, 'get_protocol_version_name'):  # requires pyOPenSSL 16.0.0
-                        logger.debug('SSL connection to %s using protocol %s, cipher %s',
-                                     self._hostnameASCII,
-                                     connection.get_protocol_version_name(),
-                                     connection.get_cipher_name(),
-                                     )
-                    else:
-                        logger.debug('SSL connection to %s using cipher %s',
-                                     self._hostnameASCII,
-                                     connection.get_cipher_name(),
-                                     )
+                logger.debug('SSL connection to %s using protocol %s, cipher %s',
+                             self._hostnameASCII,
+                             connection.get_protocol_version_name(),
+                             connection.get_cipher_name(),
+                             )
                 server_cert = connection.get_peer_certificate()
                 logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
                              x509name_to_string(server_cert.get_issuer()),

From e42d82526a1a519ff1305b189acde064a40fbd8d Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Thu, 1 Oct 2020 23:22:17 +0500
Subject: [PATCH 152/716] Drop the conditional code for old Twisted (#4820)

---
 scrapy/core/downloader/tls.py | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index b5c6cc895..2b8990b75 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -5,7 +5,6 @@ from service_identity.exceptions import CertificateError
 from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
 from twisted.internet.ssl import AcceptableCiphers
 
-from scrapy import twisted_version
 from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
 
 
@@ -28,13 +27,6 @@ openssl_methods = {
 }
 
 
-if twisted_version < (17, 0, 0):
-    from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name
-else:
-    def set_tlsext_host_name(connection, hostNameBytes):
-        connection.set_tlsext_host_name(hostNameBytes)
-
-
 class ScrapyClientTLSOptions(ClientTLSOptions):
     """
     SSL Client connection creator ignoring certificate verification errors
@@ -52,7 +44,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
 
     def _identityVerifyingInfoCallback(self, connection, where, ret):
         if where & SSL.SSL_CB_HANDSHAKE_START:
-            set_tlsext_host_name(connection, self._hostnameBytes)
+            connection.set_tlsext_host_name(self._hostnameBytes)
         elif where & SSL.SSL_CB_HANDSHAKE_DONE:
             if self.verbose_logging:
                 logger.debug('SSL connection to %s using protocol %s, cipher %s',

From f47b120e2b67be7821a06ff77786c0bab2cfece8 Mon Sep 17 00:00:00 2001
From: Habeeb Shopeju <shopejuh@gmail.com>
Date: Thu, 1 Oct 2020 19:50:11 +0100
Subject: [PATCH 153/716] Documentation of link extractor usage (#4775)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Added description when using link extractor outside crawlspiders and created reference documentation for scrapy.link.Link class

* Added link.rst to toctree

* Corrected spelling errors, moved docs to Link doctstring to use autoclass

* Moved link docs to link_extractors

* Update docs/topics/link-extractors.rst

Co-authored-by: Adrián Chaves <adrian@chaves.io>

* Update link.py

Improvements to URL description

* Update link.py

* Update link.py

Fixed line length Flake issue

* Update link.py

Fixed trailing whitespace

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/index.rst                  |  1 -
 docs/topics/link-extractors.rst | 21 ++++++++++++++++++---
 scrapy/link.py                  | 17 ++++++++++++++++-
 3 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/docs/index.rst b/docs/index.rst
index 11aa5c9be..da264fb34 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -78,7 +78,6 @@ Basic concepts
    topics/settings
    topics/exceptions
 
-
 :doc:`topics/commands`
     Learn about the command-line tool used to manage your Scrapy project.
 
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index ed32411b0..e12ad45e0 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -10,12 +10,19 @@ The ``__init__`` method of
 :class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that
 determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links
 <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
-list of matching :class:`scrapy.link.Link` objects from a
+list of matching :class:`~scrapy.link.Link` objects from a
 :class:`~scrapy.http.Response` object.
 
 Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
-through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
-extractors in regular spiders.
+through a set of :class:`~scrapy.spiders.Rule` objects.
+
+You can also use link extractors in regular spiders. For example, you can instantiate
+:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` into a class
+variable in your spider, and use it from your spider callbacks::
+
+    def parse(self, response):
+        for link in self.link_extractor.extract_links(response):
+            yield Request(link.url, callback=self.parse)
 
 .. _topics-link-extractors-ref:
 
@@ -145,4 +152,12 @@ LxmlLinkExtractor
 
     .. automethod:: extract_links
 
+Link
+----
+
+.. module:: scrapy.link
+   :synopsis: Link from link extractors
+
+.. autoclass:: Link
+
 .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py
diff --git a/scrapy/link.py b/scrapy/link.py
index 684735f6e..e70667361 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -7,7 +7,22 @@ its documentation in: docs/topics/link-extractors.rst
 
 
 class Link:
-    """Link objects represent an extracted link by the LinkExtractor."""
+    """Link objects represent an extracted link by the LinkExtractor.
+
+    Using the anchor tag sample below to illustrate the parameters::
+
+            <a href="https://example.com/nofollow.html#foo" rel="nofollow">Dont follow this one</a>
+
+    :param url: the absolute url being linked to in the anchor tag.
+                From the sample, this is ``https://example.com/nofollow.html``.
+
+    :param text: the text in the anchor tag. From the sample, this is ``Dont follow this one``.
+
+    :param fragment: the part of the url after the hash symbol. From the sample, this is ``foo``.
+
+    :param nofollow: an indication of the presence or absence of a nofollow value in the ``rel`` attribute
+                    of the anchor tag.
+    """
 
     __slots__ = ['url', 'text', 'fragment', 'nofollow']
 

From 159e2b2e2fbbcbb2b083a79e83e37cf4e60ee5ee Mon Sep 17 00:00:00 2001
From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com>
Date: Fri, 2 Oct 2020 00:23:08 +0530
Subject: [PATCH 154/716] allowing to run .pyw files  (#4646)

* allow .pyw in scrapy/commands/runspider.py

* aesthetics

* added tests for '.pyw'

* created class for testing .pyw files

* name=None parameter in get_log

* small fix

* .pyw tests for non-windows

* used @skipIf for tests

* two more tests skipped
---
 scrapy/commands/runspider.py |  2 +-
 tests/test_commands.py       | 91 ++++++++++++++++++++++++++++--------
 2 files changed, 72 insertions(+), 21 deletions(-)

diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index aedd8c2ce..b957c29fb 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -11,7 +11,7 @@ def _import_file(filepath):
     abspath = os.path.abspath(filepath)
     dirname, file = os.path.split(abspath)
     fname, fext = os.path.splitext(file)
-    if fext != '.py':
+    if fext not in ('.py', '.pyw'):
         raise ValueError(f"Not a Python source file: {abspath}")
     if dirname:
         sys.path = [dirname] + sys.path
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 2899e5f24..3e54a0948 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -496,6 +496,8 @@ class MiscCommandsTest(CommandTest):
 
 class RunSpiderCommandTest(CommandTest):
 
+    spider_filename = 'myspider.py'
+
     debug_log_spider = """
 import scrapy
 
@@ -507,11 +509,23 @@ class MySpider(scrapy.Spider):
         return []
 """
 
+    badspider = """
+import scrapy
+
+class BadSpider(scrapy.Spider):
+    name = "bad"
+    def start_requests(self):
+        raise Exception("oops!")
+        """
+
     @contextmanager
-    def _create_file(self, content, name):
+    def _create_file(self, content, name=None):
         tmpdir = self.mktemp()
         os.mkdir(tmpdir)
-        fname = abspath(join(tmpdir, name))
+        if name:
+            fname = abspath(join(tmpdir, name))
+        else:
+            fname = abspath(join(tmpdir, self.spider_filename))
         with open(fname, 'w') as f:
             f.write(content)
         try:
@@ -519,12 +533,12 @@ class MySpider(scrapy.Spider):
         finally:
             rmtree(tmpdir)
 
-    def runspider(self, code, name='myspider.py', args=()):
+    def runspider(self, code, name=None, args=()):
         with self._create_file(code, name) as fname:
             return self.proc('runspider', fname, *args)
 
-    def get_log(self, code, name='myspider.py', args=()):
-        p, stdout, stderr = self.runspider(code, name=name, args=args)
+    def get_log(self, code, name=None, args=()):
+        p, stdout, stderr = self.runspider(code, name, args=args)
         return stderr
 
     def test_runspider(self):
@@ -556,7 +570,7 @@ class MySpider(scrapy.Spider):
         # which is intended,
         # but this should not be because of DNS lookup error
         # assumption: localhost will resolve in all cases (true?)
-        log = self.get_log("""
+        dnscache_spider = """
 import scrapy
 
 class MySpider(scrapy.Spider):
@@ -565,23 +579,20 @@ class MySpider(scrapy.Spider):
 
     def parse(self, response):
         return {'test': 'value'}
-""",
-                           args=('-s', 'DNSCACHE_ENABLED=False'))
-        print(log)
+"""
+        log = self.get_log(dnscache_spider, args=('-s', 'DNSCACHE_ENABLED=False'))
         self.assertNotIn("DNSLookupError", log)
         self.assertIn("INFO: Spider opened", log)
 
     def test_runspider_log_short_names(self):
         log1 = self.get_log(self.debug_log_spider,
                             args=('-s', 'LOG_SHORT_NAMES=1'))
-        print(log1)
         self.assertIn("[myspider] DEBUG: It Works!", log1)
         self.assertIn("[scrapy]", log1)
         self.assertNotIn("[scrapy.core.engine]", log1)
 
         log2 = self.get_log(self.debug_log_spider,
                             args=('-s', 'LOG_SHORT_NAMES=0'))
-        print(log2)
         self.assertIn("[myspider] DEBUG: It Works!", log2)
         self.assertNotIn("[scrapy]", log2)
         self.assertIn("[scrapy.core.engine]", log2)
@@ -599,15 +610,7 @@ class MySpider(scrapy.Spider):
         self.assertIn('Unable to load', log)
 
     def test_start_requests_errors(self):
-        log = self.get_log("""
-import scrapy
-
-class BadSpider(scrapy.Spider):
-    name = "bad"
-    def start_requests(self):
-        raise Exception("oops!")
-        """, name="badspider.py")
-        print(log)
+        log = self.get_log(self.badspider, name='badspider.py')
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
@@ -696,6 +699,54 @@ class MySpider(scrapy.Spider):
         self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
 
 
+class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
+
+    spider_filename = 'myspider.pyw'
+
+    def setUp(self):
+        super(WindowsRunSpiderCommandTest, self).setUp()
+
+    def test_start_requests_errors(self):
+        log = self.get_log(self.badspider, name='badspider.pyw')
+        self.assertIn("start_requests", log)
+        self.assertIn("badspider.pyw", log)
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_run_good_spider(self):
+        super().test_run_good_spider()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider(self):
+        super().test_runspider()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider_dnscache_disabled(self):
+        super().test_runspider_dnscache_disabled()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider_log_level(self):
+        super().test_runspider_log_level()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider_log_short_names(self):
+        super().test_runspider_log_short_names()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_runspider_no_spider_found(self):
+        super().test_runspider_no_spider_found()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_output(self):
+        super().test_output()
+
+    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
+    def test_overwrite_output(self):
+        super().test_overwrite_output()
+
+    def test_runspider_unable_to_load(self):
+        raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")
+
+
 class BenchCommandTest(CommandTest):
 
     def test_run(self):

From 797a6690c07ae90f7a2d703832e2fe44466d656f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Thu, 1 Oct 2020 23:11:11 -0300
Subject: [PATCH 155/716] Tests: use classes instead of paths in settings
 (#4817)

---
 tests/test_dupefilters.py                   | 10 +--
 tests/test_logformatter.py                  |  4 +-
 tests/test_pipelines.py                     |  2 +-
 tests/test_request_attribute_binding.py     | 14 ++--
 tests/test_request_cb_kwargs.py             |  4 +-
 tests/test_spidermiddleware_output_chain.py | 92 ++++++++++-----------
 6 files changed, 63 insertions(+), 63 deletions(-)

diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 95a4fca0d..680bb6dc8 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -43,7 +43,7 @@ class RFPDupeFilterTest(unittest.TestCase):
 
     def test_df_from_crawler_scheduler(self):
         settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
+                    'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
@@ -51,14 +51,14 @@ class RFPDupeFilterTest(unittest.TestCase):
 
     def test_df_from_settings_scheduler(self):
         settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'}
+                    'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
         self.assertEqual(scheduler.df.method, 'from_settings')
 
     def test_df_direct_scheduler(self):
-        settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'}
+        settings = {'DUPEFILTER_CLASS': DirectDupeFilter}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertEqual(scheduler.df.method, 'n/a')
@@ -162,7 +162,7 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_log(self):
         with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': False,
-                        'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
+                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             scheduler = Scheduler.from_crawler(crawler)
             spider = SimpleSpider.from_crawler(crawler)
@@ -191,7 +191,7 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_log_debug(self):
         with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': True,
-                        'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
+                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             scheduler = Scheduler.from_crawler(crawler)
             spider = SimpleSpider.from_crawler(crawler)
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index dc5be398f..6381f895b 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -193,7 +193,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
         self.base_settings = {
             'LOG_LEVEL': 'DEBUG',
             'ITEM_PIPELINES': {
-                __name__ + '.DropSomeItemsPipeline': 300,
+                DropSomeItemsPipeline: 300,
             },
         }
 
@@ -212,7 +212,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
     @defer.inlineCallbacks
     def test_skip_messages(self):
         settings = self.base_settings.copy()
-        settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter'
+        settings['LOG_FORMATTER'] = SkipMessagesLogFormatter
         crawler = CrawlerRunner(settings).create_crawler(ItemSpider)
         with LogCapture() as lc:
             yield crawler.crawl(mockserver=self.mockserver)
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index c72f1a338..ff3af9a74 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -68,7 +68,7 @@ class PipelineTestCase(unittest.TestCase):
 
     def _create_crawler(self, pipeline_class):
         settings = {
-            'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1},
+            'ITEM_PIPELINES': {pipeline_class: 1},
         }
         crawler = get_crawler(ItemSpider, settings)
         crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index 907117468..00c532c41 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -92,7 +92,7 @@ class CrawlTestCase(TestCase):
         url = self.mockserver.url("/status?n=200")
         runner = CrawlerRunner(settings={
             "DOWNLOADER_MIDDLEWARES": {
-                __name__ + ".RaiseExceptionRequestMiddleware": 590,
+                RaiseExceptionRequestMiddleware: 590,
             },
         })
         crawler = runner.create_crawler(SingleRequestSpider)
@@ -119,7 +119,7 @@ class CrawlTestCase(TestCase):
         url = self.mockserver.url("/status?n=200")
         runner = CrawlerRunner(settings={
             "DOWNLOADER_MIDDLEWARES": {
-                __name__ + ".ProcessResponseMiddleware": 595,
+                ProcessResponseMiddleware: 595,
             }
         })
         crawler = runner.create_crawler(SingleRequestSpider)
@@ -149,8 +149,8 @@ class CrawlTestCase(TestCase):
         url = self.mockserver.url("/status?n=200")
         runner = CrawlerRunner(settings={
             "DOWNLOADER_MIDDLEWARES": {
-                __name__ + ".RaiseExceptionRequestMiddleware": 590,
-                __name__ + ".CatchExceptionOverrideRequestMiddleware": 595,
+                RaiseExceptionRequestMiddleware: 590,
+                CatchExceptionOverrideRequestMiddleware: 595,
             },
         })
         crawler = runner.create_crawler(SingleRequestSpider)
@@ -170,8 +170,8 @@ class CrawlTestCase(TestCase):
         url = self.mockserver.url("/status?n=200")
         runner = CrawlerRunner(settings={
             "DOWNLOADER_MIDDLEWARES": {
-                __name__ + ".RaiseExceptionRequestMiddleware": 590,
-                __name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595,
+                RaiseExceptionRequestMiddleware: 590,
+                CatchExceptionDoNotOverrideRequestMiddleware: 595,
             },
         })
         crawler = runner.create_crawler(SingleRequestSpider)
@@ -188,7 +188,7 @@ class CrawlTestCase(TestCase):
         """
         runner = CrawlerRunner(settings={
             "DOWNLOADER_MIDDLEWARES": {
-                __name__ + ".AlternativeCallbacksMiddleware": 595,
+                AlternativeCallbacksMiddleware: 595,
             }
         })
         crawler = runner.create_crawler(AlternativeCallbacksSpider)
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index bd49179aa..145a4e9b2 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -50,10 +50,10 @@ class KeywordArgumentsSpider(MockServerSpider):
     name = 'kwargs'
     custom_settings = {
         'DOWNLOADER_MIDDLEWARES': {
-            __name__ + '.InjectArgumentsDownloaderMiddleware': 750,
+            InjectArgumentsDownloaderMiddleware: 750,
         },
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.InjectArgumentsSpiderMiddleware': 750,
+            InjectArgumentsSpiderMiddleware: 750,
         },
     }
 
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 2f454addc..029bf8bd6 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -16,11 +16,20 @@ class LogExceptionMiddleware:
 
 # ================================================================================
 # (0) recover from an exception on a spider callback
+class RecoveryMiddleware:
+    def process_spider_exception(self, response, exception, spider):
+        spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
+        return [
+            {'from': 'process_spider_exception'},
+            Request(response.url, meta={'dont_fail': True}, dont_filter=True),
+        ]
+
+
 class RecoverySpider(Spider):
     name = 'RecoverySpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.RecoveryMiddleware': 10,
+            RecoveryMiddleware: 10,
         },
     }
 
@@ -34,15 +43,6 @@ class RecoverySpider(Spider):
             raise TabError()
 
 
-class RecoveryMiddleware:
-    def process_spider_exception(self, response, exception, spider):
-        spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
-        return [
-            {'from': 'process_spider_exception'},
-            Request(response.url, meta={'dont_fail': True}, dont_filter=True),
-        ]
-
-
 # ================================================================================
 # (1) exceptions from a spider middleware's process_spider_input method
 class FailProcessSpiderInputMiddleware:
@@ -56,9 +56,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
             # spider
-            __name__ + '.LogExceptionMiddleware': 10,
-            __name__ + '.FailProcessSpiderInputMiddleware': 8,
-            __name__ + '.LogExceptionMiddleware': 6,
+            FailProcessSpiderInputMiddleware: 8,
+            LogExceptionMiddleware: 6,
             # engine
         }
     }
@@ -87,7 +86,7 @@ class GeneratorCallbackSpider(Spider):
     name = 'GeneratorCallbackSpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.LogExceptionMiddleware': 10,
+            LogExceptionMiddleware: 10,
         },
     }
 
@@ -106,7 +105,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider)
     name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.LogExceptionMiddleware': 100000,
+            LogExceptionMiddleware: 100000,
         },
     }
 
@@ -117,7 +116,7 @@ class NotGeneratorCallbackSpider(Spider):
     name = 'NotGeneratorCallbackSpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.LogExceptionMiddleware': 10,
+            LogExceptionMiddleware: 10,
         },
     }
 
@@ -134,32 +133,13 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS
     name = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.LogExceptionMiddleware': 100000,
+            LogExceptionMiddleware: 100000,
         },
     }
 
 
 # ================================================================================
 # (4) exceptions from a middleware process_spider_output method (generator)
-class GeneratorOutputChainSpider(Spider):
-    name = 'GeneratorOutputChainSpider'
-    custom_settings = {
-        'SPIDER_MIDDLEWARES': {
-            __name__ + '.GeneratorFailMiddleware': 10,
-            __name__ + '.GeneratorDoNothingAfterFailureMiddleware': 8,
-            __name__ + '.GeneratorRecoverMiddleware': 5,
-            __name__ + '.GeneratorDoNothingAfterRecoveryMiddleware': 3,
-        },
-    }
-
-    def start_requests(self):
-        yield Request(self.mockserver.url('/status?n=200'))
-
-    def parse(self, response):
-        yield {'processed': ['parse-first-item']}
-        yield {'processed': ['parse-second-item']}
-
-
 class _GeneratorDoNothingMiddleware:
     def process_spider_output(self, response, result, spider):
         for r in result:
@@ -205,26 +185,28 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
     pass
 
 
-# ================================================================================
-# (5) exceptions from a middleware process_spider_output method (not generator)
-class NotGeneratorOutputChainSpider(Spider):
-    name = 'NotGeneratorOutputChainSpider'
+class GeneratorOutputChainSpider(Spider):
+    name = 'GeneratorOutputChainSpider'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
-            __name__ + '.NotGeneratorFailMiddleware': 10,
-            __name__ + '.NotGeneratorDoNothingAfterFailureMiddleware': 8,
-            __name__ + '.NotGeneratorRecoverMiddleware': 5,
-            __name__ + '.NotGeneratorDoNothingAfterRecoveryMiddleware': 3,
+            GeneratorFailMiddleware: 10,
+            GeneratorDoNothingAfterFailureMiddleware: 8,
+            GeneratorRecoverMiddleware: 5,
+            GeneratorDoNothingAfterRecoveryMiddleware: 3,
         },
     }
 
     def start_requests(self):
-        return [Request(self.mockserver.url('/status?n=200'))]
+        yield Request(self.mockserver.url('/status?n=200'))
 
     def parse(self, response):
-        return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]
+        yield {'processed': ['parse-first-item']}
+        yield {'processed': ['parse-second-item']}
 
 
+# ================================================================================
+# (5) exceptions from a middleware process_spider_output method (not generator)
+
 class _NotGeneratorDoNothingMiddleware:
     def process_spider_output(self, response, result, spider):
         out = []
@@ -276,6 +258,24 @@ class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddlew
     pass
 
 
+class NotGeneratorOutputChainSpider(Spider):
+    name = 'NotGeneratorOutputChainSpider'
+    custom_settings = {
+        'SPIDER_MIDDLEWARES': {
+            NotGeneratorFailMiddleware: 10,
+            NotGeneratorDoNothingAfterFailureMiddleware: 8,
+            NotGeneratorRecoverMiddleware: 5,
+            NotGeneratorDoNothingAfterRecoveryMiddleware: 3,
+        },
+    }
+
+    def start_requests(self):
+        return [Request(self.mockserver.url('/status?n=200'))]
+
+    def parse(self, response):
+        return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]
+
+
 # ================================================================================
 class TestSpiderMiddleware(TestCase):
     @classmethod

From 004b40a7193a7c0c5138abe764cad6d1d9a4fc57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sat, 3 Oct 2020 00:53:55 +0200
Subject: [PATCH 156/716] =?UTF-8?q?as=20soon=20as=20=E2=86=92=20as=20long?=
 =?UTF-8?q?=20as=20(#4825)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/contributing.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 675f55c38..4d2580a6c 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -140,7 +140,7 @@ original pull request author hasn't had time to address them.
 In this case consider picking up this pull request: open
 a new pull request with all commits from the original pull request, as well as
 additional changes to address the raised issues. Doing so helps a lot; it is
-not considered rude as soon as the original author is acknowledged by keeping
+not considered rude as long as the original author is acknowledged by keeping
 his/her commits.
 
 You can pull an existing pull request to a local branch

From 892dd9da57c5cf005670743c8abfdfffc7027acc Mon Sep 17 00:00:00 2001
From: "P. Chen" <pengyu@libstarrify.so>
Date: Mon, 5 Oct 2020 21:00:58 +0100
Subject: [PATCH 157/716] Adding support for zstd in HttpCompressionMiddleware

---
 scrapy/downloadermiddlewares/httpcompression.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 727c41466..b1abb8b5c 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -14,6 +14,12 @@ try:
 except ImportError:
     pass
 
+try:
+    import zstd
+    ACCEPTED_ENCODINGS.append(b'zstd')
+except ImportError:
+    pass
+
 
 class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
@@ -67,4 +73,6 @@ class HttpCompressionMiddleware:
                 body = zlib.decompress(body, -15)
         if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
             body = brotli.decompress(body)
+        if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS:
+            body = zstd.decompress(body)
         return body

From c6c3f2ce661ef9296302b9d489d5a2cb2e49f481 Mon Sep 17 00:00:00 2001
From: "P. Chen" <pengyu@libstarrify.so>
Date: Mon, 5 Oct 2020 21:10:40 +0100
Subject: [PATCH 158/716] Updating the doc entry for the HTTP compress
 downloader middleware on zstd

---
 docs/topics/downloader-middleware.rst | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 06e614941..9645ed5fd 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -684,11 +684,14 @@ HttpCompressionMiddleware
    This middleware allows compressed (gzip, deflate) traffic to be
    sent/received from web sites.
 
-   This middleware also supports decoding `brotli-compressed`_ responses,
-   provided `brotlipy`_ is installed.
+   This middleware also supports decoding `brotli-compressed`_ as well as
+   `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is
+   installed, respectively.
 
 .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
 .. _brotlipy: https://pypi.org/project/brotlipy/
+.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
+.. _zstd: https://pypi.org/project/zstd/
 
 HttpCompressionMiddleware Settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From da3171d4f71d50a15b08b76b2b1e06bddfa56694 Mon Sep 17 00:00:00 2001
From: "P. Chen" <pengyu@libstarrify.so>
Date: Mon, 5 Oct 2020 23:18:58 +0100
Subject: [PATCH 159/716] Using the `zstandard` package than `zstd` for
 supporting frames both with and without the content size info

See also: https://github.com/sergey-dryabzhinsky/python-zstd/issues/53
---
 docs/topics/downloader-middleware.rst           | 4 ++--
 scrapy/downloadermiddlewares/httpcompression.py | 8 ++++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 9645ed5fd..7c63a623d 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -685,13 +685,13 @@ HttpCompressionMiddleware
    sent/received from web sites.
 
    This middleware also supports decoding `brotli-compressed`_ as well as
-   `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is
+   `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
    installed, respectively.
 
 .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
 .. _brotlipy: https://pypi.org/project/brotlipy/
 .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
-.. _zstd: https://pypi.org/project/zstd/
+.. _zstandard: https://pypi.org/project/zstandard/
 
 HttpCompressionMiddleware Settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index b1abb8b5c..56421a6ba 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,4 +1,5 @@
 import zlib
+import io
 
 from scrapy.utils.gz import gunzip
 from scrapy.http import Response, TextResponse
@@ -15,7 +16,7 @@ except ImportError:
     pass
 
 try:
-    import zstd
+    import zstandard
     ACCEPTED_ENCODINGS.append(b'zstd')
 except ImportError:
     pass
@@ -74,5 +75,8 @@ class HttpCompressionMiddleware:
         if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
             body = brotli.decompress(body)
         if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS:
-            body = zstd.decompress(body)
+            # Using its streaming API since its simple API could handle only cases
+            # where there is content size data embedded in the frame
+            reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
+            body = reader.read()
         return body

From 50e1f35d1fcaeb2e0a0c4fb29894cb7c686a33cd Mon Sep 17 00:00:00 2001
From: "P. Chen" <pengyu@libstarrify.so>
Date: Mon, 5 Oct 2020 23:43:12 +0100
Subject: [PATCH 160/716] Adding test cases for the zstd content encoding

---
 tests/requirements-py3.txt                    |   3 +-
 .../html-zstd-static-content-size.bin         | Bin 0 -> 8066 bytes
 .../html-zstd-static-no-content-size.bin      | Bin 0 -> 8063 bytes
 .../html-zstd-streaming-no-content-size.bin   | Bin 0 -> 8047 bytes
 ...st_downloadermiddleware_httpcompression.py |  26 ++++++++++++++++++
 5 files changed, 28 insertions(+), 1 deletion(-)
 create mode 100644 tests/sample_data/compressed/html-zstd-static-content-size.bin
 create mode 100644 tests/sample_data/compressed/html-zstd-static-no-content-size.bin
 create mode 100644 tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 2247ed917..2eed2f5da 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -16,6 +16,7 @@ uvloop; platform_system != "Windows"
 
 # optional for shell wrapper tests
 bpython
-brotlipy
+brotlipy  # optional for HTTP compress downloader middleware tests
+zstandard  # optional for HTTP compress downloader middleware tests
 ipython
 pywin32; sys_platform == "win32"
diff --git a/tests/sample_data/compressed/html-zstd-static-content-size.bin b/tests/sample_data/compressed/html-zstd-static-content-size.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b5c2038e893481e2835ac74bc790e3718986e426
GIT binary patch
literal 8066
zcmV-|AAR5`wJ-goRU`obmHPmitfU$&;EqO3lFJAV16>@^C8(hY1ZsEGeDXVhj51gD
z{wbwTMW0fDVIjsC8zUwJLIPd^Tmj940%8FN4lqZfU`(TE6bp>O3<NNZ!9*fXdqgZ6
z#ezeFhorGg9uWuA7)^PA!C4|oqcn{|lOW+VG+3C)#Ay>tV-7e>YaR^FV=)2Knk9h*
zPD2Bxu`I2bEJ}oPC|O!VVVHKB!!*Tl8rtL0DmWw>I4vT=w8gSua2St>L(&)r1jK1r
zB93OkEFwfW4Gr3a353%c%i_@@6d(<P!D%cJ2oDuV77q?{AUr%cjiWV^qqK-cgwj}A
z!?Xv^GMP9_YZ^tev}RExjYV0M1p{#=6Q?yE4habpabSY6w8er7r=hV-7>g3YGzZS2
zoTW`Hi$<b2IBjW+rZ^-dOq<{^E#fS#u}m6?(jF5iKo$}-O?%)p3Z$86n&h-b!nD&A
z2j@6666G|Dqdi(<S)2xOES3qA#e@lwM#8koqJh&M3y4C&0<tVlQyAwov;i8*V?wfE
zfCpp}S->=w=8$L-PHPy)01wDyvS31MEJ`~NhjNz41g9-ZJB>s+5DPeP;2ej=V*+6!
z2AsyCEXzc~31Ax}lf`M!Fs)Hq1*RRwGC2*?lnG^m!iYdl<B(9=gTY}eisVR|;sDYf
z6blXkP*5fs5uC=NFd`C5JI&IXB_eSgjZ)z-TGJ>H$+BRYv$Pg*Bn!uBmIj$fL>Nb+
zI3N}nq$wB$#lixsfDwsMEIcR{9u$&CQ<jB;BZ0(1f#Z-^EN5wtM~hfMu=Z$_Msdyp
zV=S6OvA`aX#-UhX5F$Vza2_xK0Ym`{H~>HqNO3cLt#DP1=;z9HvN0E*SB0B*i|V@?
zo!0dgG3lazM?J5RBBIe@s@IB`{`D4B*O(MBjS}6Wt}`j>9cd;_G)lU*(nS1E>Agxb
zjnrOTx<<xbn&~^eyS*!_bJ3}K5x-7e-w*w(eCw;xLVuUnzM|rl7ytF)`qx#f!^L#t
z8vj*)(bd<8`J-=BN?czjuJxyhj0zQ3Z*M{f@fMHkA1(U}^$;$)uTRzeE#g`iJ>i{{
zuB!54rdz~ydTv6L_*K<h%&)q|g=_VM$VrcK&wIK4;i_DeF4L__d(n$u=Q?*?SD`L)
z(x3Nq@tfY;%=O*EFY>KIR4?Lcu5WP{s>|c5E+#Mj_g#pa>{ZT1y12Y}n{th}_Fg9A
z1<<e3<TWlDSLM>pTU+<?n|!$n;l6oLoWmt!U|O>@2?hd|hVob(Mx!|A9L6+?2gL$o
znujl=Vp?bP`Io;tI|2zX28DP;Bu<1Tv0&gpczD1Z49GyhvY;TT5Mhqv;Q{e@YIOW5
zqWax`XR5Q;t%z4Y--^b(_2;8+t;tDU>hpQ0m`+OFBE|bpS6Zlk`l0deZdK^3BEqA>
zRR2pCDs;Ng>RwcLUCNkuG%jD}5BD@tF&)x8I_jk&p7V)G(`gZ{6xZdtwNm_#i_Tl2
zqB?K&qnD;qh`TiXT=d>C?`4E=@0jSv^u2Ns{X||@`<aM0J{P_C^h&vCye2}V^2kdU
zk=OY|w_@fKc`4qXc|@<DOVMds*Tw(2lJ}W-T|AV@x^&Tqj?XVX6W))xi0fxwJth7}
ztE)UsXx-yoeNorP!)NMxeP#Z)d3P(*m#6pJRNTu{iFAFZ>%Yx~m|nN3bgg-Nx_tMh
zb2;^SUoY!f553Y~t6QVm_vRBZ^WMz=)_7=tdGooqRzH&-Za(+c=|tQ_UuU{hqXP@<
z0cji$4~m6_Csd@4D6L2Ibahm`g!G*LBSHufrT@5&`E=gt9~CLC;}Jc7L|=%~6FWqh
z)KQ{@tS;?4N|cF5C{yuIl!%f;qC}+9QL>bGMwkhqyiClabV{lyoz%Zjs#H$uh%&8(
zGE+h*FQsBA^U|pZX{r@dh?4q3dQWF3rYIBo$(bpW`jX0&^eGck=d==2g+9fQD#cPN
zrs*jaN-9yMJ4&e7Vo6!qdOH>KdRm#P*!oJDnd^{Z>vwA1tum=Ey;6m#)M8huSXwL<
zyYE+pnzz`xLRIF)^nSQ}mAF#nVoIqle<}8*>gtp&UCI#3`$D>wuEmgpUg~u*WtaXF
zV@FG=YEsvw?77xb#?qIJe2AxK>MKiD`*%9?=$NK5X1y|B^~%4xGPYPE{XHF$5%uVv
zh$RPg#J-Nu4u_8E88LN0hKV6`NVMXQA;#D(2W5Cv4reS8VvI4C3^^pk7z44y7-P{K
z3Za>!Q7F?WBrt{yiq@z^(6q(i%pd>(0YCr&0Mioy02mAmh>Sx5DrwgN6o3I56b29!
zEDRI~0t7&C;P4u`C=>t&LSZl%C>~Ta1x^BJ0QDms#I{8t+xDvw^)H%OmM^UtD6u=U
zJi2g;L;Z>qIeYC%x77AvCUM0~2dZsvqsQWjcw9QI^3oQ^-Ny=pBn5wXbuZyGOE-C_
zU+(88f@qO{Z!@13Ld9B{bEg_PV51mS_}EE@qw|Xtd&KjUfOP~dg4w^u)KKYBv)CAw
zot38EujidM6)6_Q0emIE5v2Dd^J_S#jAd|478Ri)!r?XqW)yq}yGjGyFZpbke@%9A
z_^u{F`#3YR{645{L-~!AtK$yGb6vk!62X2+S8;$e6dz#?0lkbchOLSGA;3MZNDi7_
zDZ`<`7%Pq~Ov@c1`J>Q+#XEwZ=En{ju0tq)SgwM`F;(vui(^FjAxj@&3Tt+_cON8;
zz{D;>Q?B)}jJ^@EIH)ri__uIYjQ$Y@q47ge6H$0uQ(bqJX;zw;fYKs74qQlWnM$zn
zt42$TaFlT17RDKQ`q8l82lUt`FP*I7d0CAn3p5-cre}g%ko2+aBjS01-TT!%Qrs`2
zL~I4ZXS$tk!Q*FY&%tS<K1WbkU!_yR?pOQfkr^`r%j#ppl<pSl1!cT}XzN4PC==8!
znGl>qr}IP3GN6x2d0K&-ZUNiTB0Ivy7Z|@j7m;$}>s<S_;eHRWF%E(p`W7@5cZ5@o
z8R4+XuWyb2iFF*RYGwrfn8Go!<8I+@neit^&z@Zjs*C8zp0K(Ewf3WYd<*}y;c_K+
z@gFW%F0d&2|G(k!soz8XaoT5B4hnDc-o77!;R^yA1%ZglSLs2^U=4-1sQohBR?@Rq
zNj(>J?+$a1=&4s!@7FS6X}iQx6KvgDT?Bz{7oi<Xy&j6?Tl)flHiATul$UfT5P0wN
z-c%(Kd%Sf00-4lL^7sV!!+TzW>lbL9z-4ct_rAsPzI!Y^_3E}j__m*@@3lVL2ujan
z-u(=F9dJ)WDyvlfO1+zK;~Mmg=xq%#`vU6msw@-M!y$M-#>{el>f_pp|83onyRevw
z@EiId#6snzKtlyy;h3<xI@$c2`lVWlxe{yrKzQh*mP)nnw0jCG!$I`(e<be<xd_eW
zbOg9jxBe5tL=b1UT20h4_NOHLAz<3Y!u3I7vx%_tWWeu1ZH5tLj58c0>0OWXlx=z~
z<wiZf1_=uBi*Q=lz~8|BhE`n$y3key#SShak1qts&?eKORUy^Ihdpi>8}Cu#3gc>#
zIsI;^khN5tfp}$v>?3qU-7kD0p8a@sT=af|0#_@H>GQR&&9K?&p^xx*zgWVa9Vz4P
z4;Zh}^^L+z-surWhn#H`tOz}3p7HvZ9Kw{D2R?l=7)&}Gyqjjo#btC!7Syv>l%_4S
zLwFzbs!lxJBv+9Xhv{~*+wa~&-|>*CZ`3;@nD5bjTf>=$P3rs}kA#>Wz7~BbuZJv8
zndrY)4{tW7znl}>9`ep3Kh#XT?jiGkI#$4%dBoD`P5%KpD0O2=r;|Mb<&a&A2?<B*
z%=!(}ci^7`bz>8>sh%T{-p_2KWE-H%9O1BV(V`{m17l>qV~z=?BrOh;_ok7@rd2|~
z`IqZ~j#yB*?T5UQcoG{c>rM7YkW2>8;YPtzq;4M3q*%5%@@zBzElOkE`!xUm@V@_h
zF-3O)mbFwQ7h|5XRm0Ma9d?>`gzZKP8)duDY^I3gof79a|Lels+-KtuK4VlhC3Ih0
zS@X39#s>(w_aSJpT9w=&J}k<JTfZY{)igUAY!O<u`<)xI5#DB23#JjZ7u?rUA}I-K
z>{$ads~zVPbbbhdHgGLH>|G%lp-{0F#l;doyZWe3rMJ*W&^`p7joRO!v022)e(S+p
z#yehuj3s;f^${>C5j_r4M@v^RvCZ_?fQoR~@4}WtcGlRGBi|pxdi$syZ}exD$01+G
z!M>kq1o{kDtgmOKHpPtot=pS5O4zuF<rY%Ul|_G+wr8FG4C+6YdMkN^uI<P-inj<<
z73on-unTyG_?g|WX`Tw3JjV7%{d)9>=()lO#Q8LVERrYO{T8<%5qO!rCdcTd9Diy&
zC!{vY%$8?enEN)b92Nl>-FXKbAROmvSiFp3vPTh4Zp%8Z=gH<dt?K?L@ey`UsJl_I
z^=-Mzc-=Ofen{WQGK~?`Li<$iHK%b!e5Od1V`A1qMzr6CdsHwJc?z>J?K>4LnS(xX
zb1J2eq<evNM!Vg)Eh9BGWsgupyEbFer*ay%G|2Q_`9?`02fsThMS0fYxD}?NIPh&!
zxn7y20+`0b(DDJuGrpYy+>R#`H`qoU?o?WSpX&zX!Zl6h?>1f9m)UeGc;iMW6RBN`
z<p|sx)j;&c^ez*z-A-?uW~;v#;^yO!eL(Tt?-d&bxVc@HB>kl5>1k(4{f2WZS$^!@
zVpweBLB^Ep_DMvN#Cg|2ESpjZP&ZnkE5Kw{5h08s;?v_4zyDJ|M$~D7c0cAczQA*4
z7-2EajlyM1j2A#K9ukMp75cR6ap^JDiQXLOfGn<J_mxp}F{)G%zEeqW9l6rnSeT5d
zI5YlYE!Zfb#l6b7bM&m0%93CTf!rCAHl}uFNXFXw#zmdLAZalrsrrW-vrC`urZWDq
z;n77~V+tbYPgiH6ykJ={ih~qmcQLgeN?uH{FZN+fAGXUj{3x3azrl!4i<3t(wjUC;
zRoKD|snY|^F~uBxglvLu)J_p+*S<$BM7q;SrAU4$y;o;UV!$)!Aor>P;iKT?t%&rN
zW6;7Lzin2>bV9bWHd_5g$q_v)YP1k_D3ry?S7nHbVHw+=Vu%rQXzPTumK-5KS(A;&
zI_u;DQK;-3XAyWu3o+}Y=g>xm!&@972xxn$eg8vt_ardqgyqTVVZNKWmp~shY`27o
zq69i#6~P3nHkYnM3*&Hgg-N^nKa0yH`ZIH~mG|}O!Cn<_vP~wR2{kxG{GrA*RvY8K
zNliJ&ZTjR9^!=>yVcRHcZv<bjM6+Z$Wlr#TebP&5RjH0y?&}I*X9@qa>-zA@5v#vp
zYI;Fe6~Qj#_X>8S-WlaX9v?%^mbl-kD|H4URprK77}T9wsfA$5#tsiB@w$k-t2hKB
z;ckC-UZ+?VQq@0oR}SiGfG>2g(5`^|$T}f|n$P~Ac@h%RoBOO)QL0U-2!P%wips{d
zsLED_O>Q@N_{5ZgmBp$Ed)<`1hO#8uDj?_B5&nnuv)yVNfeq``N6E(e$>H=yi3F5z
zzy_?9I!OZLvNVpiytoK=mlJ$ulD1hq-oO!53_>++*WX4Vi1}7lS2-&}5bnJ|t~%?P
zMTEb0rtS#zI2RWRHDwG}h3{y=&8>Qr>j)!3u*(hn3>26YL%s@o1my&q359knG`dqz
zyk30)IO4i98PPUCphScEb9w~$piA#FaT3dUWv)o-pKBElLM2yeoAZMUpHI8uN>O^v
zQO}LjuKd*pH2f1}na}!3u<@8A)Sw#kF4XQ<WV@Nib<=TT(o*v158z*6xoPegmyw4v
zv|@*!Wxyz`2v|@geKbh4HxhpD)VUB5v3AWBx;JiMVa4%$=T$k@4~u|ekoqx#dBMHm
z7u0;+WskgV&g0*xT3ML~YdXEL!w4j``#+w$`TEYr8y-0;EnWuKf??d}f$@Cf-QL+P
zt9j0_ZchCC%CdVaPlF<7+Z)$xEnGTQAL2Uiq{=Wy0I&JUo>89j;t)w(`+N4DWmn?d
z2+i9@u~Dq`ZcPfB=IjZG?h=BI+}}+Lkz+7!EQf@RHL1(nSRA_-oF9*2U@q;<so^n4
zmbExqzCi+_K3PtKZ#Wm4ppV|_>UN@u$VVQ8KO9Bv<v!>r|F6nB#ZEtf8?B^8gnu^v
z`WG&m<dEJdQV)M|R$#1(i1&*QabpZ?Lo#!h-e6P-)<O%139pDKc2Z*65*)n`H1db>
zKEQJZfNxH;;~fY175Z-j&m#Hdqmg+R-GHQ&gD9o0HrWqwurD#R-ySmGv-=HI^c1gD
zwYgO}Sti&zRVeh`-zlbaJxPTod6hmnHB5!=sR#GjjqI&<cK`Idy;6Oz82XY#<<#=n
zM|C9+n{t(G)KNiGh{sInh1w_vDTJ@4k?Uz6a+`KQ?=5v(KHftIqITMf+)g2pz=zbj
z6HMk0zTLHP1*-9olboo<i*jL_9mxzYHx=6DPsf@hbS0P=9#7QoFby=8<X*95{^O|s
zMnoY%JaD82{WyJ?EU+VIDj7bPb0L~#9>I4d)OoY{jCiG%^9l`AFErDd9JHjyE;%@t
zoSwtaPOn<A|KnV)UU;oNdAEQsL1<o7`67UI^z0;I9R9hb&!9b>#wVgJ$`LE{=$fGi
zEM+B+x6g!hLK!By3B{`z+qL$?*CYWGQ)=tpGL@Ll5^qV$>QEAfhw)z*nPM$S6o7Ih
zPB}WUZ@V^N5TKb_H){!DoiWxV@iZfO7$#tj3H9|I10G4Lhw9o@v5pZbX^G0yBDE>E
z|M1D4{8|9ZO#=&)kYbd&S{&&EXvJ17;={#<Bp?0z!Mv0Jomo<s0+(}8Sj$U=C%{fV
zc8)|z(ng)*OI!RG(iye2d`a{q7!w)K!+-ZlVEFV=A8cD}ZmgAmDawZ3B@t@1a}qyH
zkn$|ql%v}w0YON61_OeAJR}2gyA*a7*KhQPIjaAy<i)&umxn<6lf+)2V6a()0#*vz
zEVy-fM00Fy%CQ_aXZ&<Pd96_IgLRl1Ifu>0B+H$nB)%W7rnMddqR5PS43SNe;E;F9
zE+QMlb_G*0toS}$m<jAxWoX!xgYxIyi4RMDQk_jju_IiEH7sBCB?)YHS7$6VmWQVu
z%#!P;7i_AP>Ruj?y+v7epym)z7Z*RiJ!kDU<rqKgxYmR{e6T?|03m|>yGx!0q#goF
z1lU@}erQBm5)WJ5R6+dPE02?nDoLSL6FPT->dG~d1yVNUjUUQs69|>6qQVy4<wp-u
zGF8m};!sR>P6z8egty%z#uULZ&s2jW?Fp4iQKq^Wz~HyVK$F)P4V7DLN8Y9!+F`+s
znMjga|7GOl_+X&4Yx4(;8>S(-{XLklu7{62fB4o9#sA7TODSq|iYeuGFw-i$ALJwN
z?YCcejmWGw_gsJF8AQ>ZjFHOfvhE@DzBc6g;L7?@tUm03-g@0*e(pV7BHJZZakVL5
z3tSnbG5*j7S#h&Gcvoe0R%iolUJz&v@cCRHj!~e_TnU>%gmH^RUWYh7Vn#D;m~?tb
zuU91Uf>ZwMkbrR1IxN3DxJybgk`Q>`i<jTe<?!(_=#Z&mobs2?)6&gp|L#cJ`y*Nz
zS(Y)3SZLx3ZO>CW05&3=^63x1eOF14G3enY>+)Go5O#6<Pn4<DViVps@8Jx3TvNCn
z)h-F!;eR$ofrzI>04MLGWJRHEUjnhpxgKg>cKb(@=Fvn|J%$(wPu(&hPAF+r8GnOx
ztum~If0)stw2OxFuNI4bk5~l2m}^0?mriM`sW;69uY9h<?VAS5u^YkPOOJ=<PEP=K
zG<2fV^|eIEFJaxB564<2p(p>=Vb<CGqN73nJ2m|3$h24E-u>(Cf0*U2sy4{>uAw%B
zmE<e@mRp66M&2nFDxAt@-9wC}1NqIMPR%`WgAi8sm>Q2}+14&Cu-OiA%TFJYs#r*A
z)qynN(DPUDsJJ9+gOoC+Aw|wc5S6{q50eQ*p^r`s1kvFnwbU|(kkUi$93rM%zaD9d
zKM;g<d5c5Y_=if5MMl7(J;NoZC@-V9YFVv&2m=IOO43raG0fs)At(#HA#Mhs8>Fhj
z^O4=x(ZU?v!;*H1)LPX`2>=bGQEkTmJFYLS`d8S(-k}egdju$|jTk*FF*uV6U_@|@
zF67`#tjJ{MOP91@OW<wo%(XkjW`KRrETs&_*s=t{{pFly9@#n_I;vbfIHf;ypOzRh
zMHyF+U;)~65}4v8fggc$PbqW3ENgSW^juphB8N-v%`&QghHSOHDtwmxDb!{qP{6_u
z6?{$~8>2{?20f)Y;;NuYL<lA)oK7OiTEAq!*__vXcwjSwAacE=-W1G`XAAxl`;ge4
z4Mj-B!z7UN6%z%Hx&4YVtRshvLUD)%fi$eVY)#2-%by!mWIxa6L%eE_-AAjQ=Y0(u
z2^mB(HYbUtSLHZY*sU4Sb~9VpHxU~`s`ZsAr6TgAxLWKXHVaC6Pe%J(7sO&4dsM5l
z++_xhP{110xOpsA1+l>Ph#Mog7UaG>(aKx2S@$}hMGhrsiA+qxmf%?3EFI|SAoj}a
zXG$P0BuNp$3bkPn{;4;MdRR7kMEh=P=&L&u9D0*M-p~(25W<t(J86W-Le~JXWH&Su
z+0cM>EBby}_n^<(CAU57?P`b1-VsB6%%MEM$O4iO*V8pw;o%2mG0`0C^h>T2oiL&;
z4R&bCN1-EepFzb5&1BdoF~aR!R0X6}B043_OI3o-I+&2q(r7uR4r0D>!=QmuFB_dc
z#Cd;!C_GL`*pmI&4>q!5tu`kwjfe|zTvjZtTG1)=g;$!8TSusve#Lmrp?xxO6q0;2
ztc4FD_D7#paqN}5hyVE8I#)3PpWNT+=kcq#uruz8Pg&StbtUaE1D<Ed@dOh<n_$6L
z7vA~k?kaMy#<HxRO2-`jpIqBCXgE;;vltNOViLfd6~V%9UGv*-!L-xl1TFh_)UmlN
zhIVIxvr=F_dT~Xy@7Y?ep*>v%t&A)%2qlZS?rpybhE#F9kknmP+7Fn%e(p)Bikzjs
zcR?+z>z0)1fs*;ryQ|0mNP;L{`(hO2G*75pX;+M>D%f45d<|;iFVkU)W7BD<f{1|m
zFoV*_f4ZfFRt$fzp$NE;ki!2^BZYr-lq>dCoHDZMKHGZ}fDahIn9VA30Ww&x=<r0R
z7Kh&1&k&^?H+D(lV5A~umj2I7WNJNba_&c$SMhl>UM=SKb$wa>@yLXOClaN6gNTHP
z9DQv@Q1xLW;JQ&`4&{bmhJ4BgRg(**gUYT=D?zte@jkjMMa3q3vpvVv^M8SRBpUl2
zLpemb6)#?PfM<SEqdKbfb4whVh<Q{~2}Eq@H;h-?tHt8{J^bjt6}fQ)m1se-h~E4X
z9!z?W0MH0Vglm7)q@y96=Zc0{-3=8!kP>RYM6i*kHj+U?%`$pV0?7{x{_$IY_oK@u
zkRPiRQarhDiz4%x$WRVtGT*2`$(^|mp!j-<>F~%-hSAHL!e7lv79~ymqYD$oGd}~T
z=R=M&7xyD#K)|>|OF%<rL7&}16&G3!zCV+&$p~>;A~C=c3?=8w2fzC4ql1WI<>&7|
zi6DkQ?l1Z<9A<3{Ihn03Q1PPFrcD%B04SP1R7AY5;9mXqSsB}?!;SRcQB;W1l1+49
zCkOOzJ=c@$Gv`Yyw%v9W&-~bM6XTwaveGIflSPj82XY4+q$E4hO4k>PuzU$I5*T=*
z0xBms9%iY1P9R$;^u2JzEj2V#T(%A=Tq*u^&kdP^-hMCmgHpsS%<+F6QqbD_d<^+2
zq&&gVmRy3$wW<itnZ>Qod1t5diDF+o(>iFdl+UU@Vu{EE6gwipxWqEJS-4zp>)t+s
z+CmgEpIf%5c<L-`Wq3_KsFb12V2aP;E`rw$LRRwOCMeRHTi8SeyX01Ye&`-SM2dn!
z*2C1}5TS|=+*++HY!_4iuToCB$B<HBNTw#Tglc7FaAv}aw<VG@B6(>VC5QEM7S;&F
zpNMBaX`fV?!T{488eyRFVNiT%K0z2<tMZ{AA7BG18F9PpnEc<xvv*C$6ksH(icMk-
z3--??ebkc_niN%m-2EoFuthT^qMZlOhMP_{R}J>oY|(O+v~@Z_7<G0V4^e>nW}3Nx
zmCsSs<o*gpMI<wd(4X`4mCL#xhr;0iH!z1Ji&+djC)DPty$JXQMZL)1LoZ59rr;#g
zFdPJXVoOkr10pw!dS(6h)?x!`*>xh)?|OR~R|-qkyEAi>n?Lp$x2wsrgVwa~IJ2yx
zg)qj0`!t6(&At=G4uAmQ0mNC-E+nFiNfqr0baw3p?4DvAIAwv&#u3OmDi(ND^by8T
zEVK<bTdEZb(XHKYWtVN1D3mfh3C1bg$r}U26U&GNs8=Y~>XBZi%IJWhNr%lMy*J+w
zbV-M7DA?|fpWc_gx{Y5?yTCy_hgBu0ex(2j`f+xKbP1SEm7n*Pa1r{74DruGNsbVL
zTw49`*DnkEFstbn-5>JUWa{LCm0L@QeZ1cOrfa?3T5F%eR~bs5QGO*Wckg~uFtwW|
z`Cm6sdO>HzPn}}t7n9^DRpfn%JJn|rX!jD&(yLv~rmxBWEZ6n(3m>aN+TM0m-<k~L
Qd^^;*p>*t+9}@VeBC4QW82|tP

literal 0
HcmV?d00001

diff --git a/tests/sample_data/compressed/html-zstd-static-no-content-size.bin b/tests/sample_data/compressed/html-zstd-static-no-content-size.bin
new file mode 100644
index 0000000000000000000000000000000000000000..3d494192e2c72294810309eb14e8d55e0431fd1c
GIT binary patch
literal 8063
zcmV-_AAsN}wJ-eyIF<VVnyjQ6EZ~ktO_Iw94g*~r(Iu#%2n1?()O_+gfQ&L%_WmiQ
zPeq?nfMFrV7#kxd1VRE{0bBvig92g!2M#btqhL&<XcP;K!3+d2jlo1BPJ2Ww8pVP`
zgNLNCOdb&j(-=*8fWcWJN~1K5LX#ljG&ES4$;4?BOJfc=Oluwt&SNnF)0!oL1WrQ(
zrm-xonJh|#b0}F_Lt&V9n!_~3aT?m=(JDA38aOQ?!nDP*U~m|Zh(ppC2L!}vSR#&Q
z!7L&~I1LTjgb9Sx8q4C*A`~DEg28Dl5eN?zNEQzcb09oCIE|w<lB2YUMTF8=TEnyl
z&N7)eOKTcMvb1JVB#lK`lm!EECKIPM9u5f!6LDaIv9!g43a6p5Oc;w2!88ZXqMW5o
zEQ>~>I5=%-jHWmwButy&FfHONt+7lRiP9buC_okxG);TpGzz4dXqx1-M#8ky6bI)x
zG!o@Bi=#bSV_BRAaV(Yzlf{Gykw(I_$)bVN9t(&<!2+@@PE#1?G_(O4%40&ZV1Nf?
z5?R1BmgbOX5>9Iv#{dt=WU^pFYb;7T5QlP>$ONY?N;{22IS>mtaNrz=#bW|tA_kns
zqAbfq!U<p-B$LHy&@io0S_P&Z#xgk#)07Eig2ISEPUDbJ+JnJiEQ;hvn&JS`9ux}>
z0Z>pT8WEhvqA(&7OFPZdnk6D}9F0=pFj~_n5XrJ&nzOVPaU=`JX_f|=NJJP%qBtNH
z7^Eo}1;xSwtAG)SP%Jzs79JFmM^l!CgCl{&LV@FuSS)90k4KAGK(O{`ltyvR0%I(i
zL$Safkj9}{U=SieAaEWq00Be+3pfBk5lC?}eXVd+jp*mfb+R!RpI3#OcZ=$~8lBel
z6*1|ee@8v9ks_keVXD`PnEv$^Ro9plF^v-4qOLP3>K$n&O*Bfnw$eoWPwBl%GmX?<
zT)IZaU7G1Ty}P|Ds&mn)dJ(@)Uf&P>t9<LL(L#Th*S@0Sl^6f@;riEAtHZ@~;~M`}
zf6>*~i20*$Q%YQ4C$9CUiHr&rS8s1Z2=Nw=>mM!q3-u5#y01^w{Vn2J7d_#fl&-4s
zVy0Wfb$V_>l=xNET+FY!#f5A2gvd#ca?g9Y{^6=zlrGb)N_)|ZU*|e^U00zla?+po
zbn%<s+syUd!Y}fzLR2r}YOZf_7plwSsxBrk{`Xyoo9tE2MY_1Wc$;#KxAtBp;|0*K
z(&RNR8dv4g&0Aad@|%3Q3gNzaP@KahV_;gdGzkU*mWJ|J97dx!=N!f~iU-94W15FA
zqheZT^!b;+J39giFb0KqL?ljxCb3}PKzMk-91O@nz_OqqsSsg~<KY4EcxrU~DWdw_
ze`l(**R6<GKi`VRy!Gd!Z>`BmUF!3Br<hJk-6F;NPgh!~e)^&D?rv4+t0Kaq!c_lD
z7b<kR(CS`PcU{VucQh_v<`4HYQ869TJUZ&7BA)Y!Nz-W&trXYgy0udLkBiP*p`to(
z^`n=jQi!`W{ap0kG4ExBaPOGt$Mn5&5&cA7SNoZWH$E4=`1DG-XuKvur1Hp17m?Tb
zM7Lt*6L~4#pLs;DpG(ncTGz$@xsvypcwIb{$+~pWh>p)MJ`>)Lxrpm$UOgrLN2{wm
zO=#WYU42p4$HQmpdVOX7w|RFf)0e0B+f>}kREczbr|ZAXg_vHqsdTM*d%Ar0rgJ&<
zd0#K<Sr5I^U#nZA+V|!YG4tNc|JHbDe|ht{w^l!s9&SGO*6BptL|<pRRHFk6>;Y*U
z4-blkg(p;`jwr21^mKJpyoB_e{v$#N5vBjQj`?)n=^qs-uHz9se?(u1(i1yGnAB0C
zgsd*@J4%#^M<`SAPn3v~L!v~a(owRMcSe{Ap}b7YqjXBDD4o>5P^wf;>WDI}g)&n@
zC@-aADD%>(2x+PnQ;3rKLV8bUC#EP9`pKCollqd%l=LYRQs=Z1Q-waokSfJeDyHcv
z6-p{mr8`Qf*kVap*?Kz_^LkpDs@VEUnVIX5V(WKm-K{dIFTGNQsnlXusaRSp6}#_O
zg_^h6x<Xav#q@r-e3iIT<zh;yE`KTZrRwUGEnUhG%KJjPmafH+gI?-&F=dzj6Jtk9
zscKT!rR=%ZQpVDkjC_cvXX-0UR{M84^XQnSGG@IpU-inrx-zy{BK<ubk`eXjo`@v}
zb;Q1o&<=-==@~I~K!%ASbV#(~k0Hj`EeB<IR1Rk>5n_xnmJB&0#25py#291I915YC
zqfsc+C?qh342ssMM9{Rw;LIQZ0Rcb&0RYnz000;a42X<F0xD_O0Th4%8WaW)6f6uB
z2m%B^aNzJ7xhNC>20~#l7$_c8GzCrqXaMyi9mKXpA=~z=5%n*cSe7rX87Q$kvpl+R
zi$nd26FGbBNw?JYU?y?JOb4oMZ==WJiFjN(t@6?q$KA&YgCqricy%w~G)p&ms9)~q
zCxU2^e{VCN7DB~ZnRBNaIbfq0RruIRhoke06nn(;lz?>vErQv<#?(;hQM1?>m7SHQ
z-mmAKHWeur#Q}UJz!9YPB=c)Hr;KHAOcoWPBEsP|1ZEU`2fIoG-7ooUn14-narmw#
zLHjr}v;01&ZA1Bul&j+o$8%l3SQ5d0Nmp@zG!!3U4FSE3Fovy({2{<Su1F4=UMa(&
z!5Ax!ElkTDA^D@wg2g+6pXSF78?Hkre^{=9#xYgz7mH&=`5{XmVG3(@xOX2Ujlje%
zLQ}5wu#CPDu{fwR82GnvR*e1;2BGmoQ4>*kTT@+km1$O*n1IqEJPuq)ZJA22@vBBl
zig1*0;1<RidHT_?-v{*ACNG_=;(1w(CJQtiAf{)6Tafgz>?7iNf!+JnJW|{*qeN^4
z!e_djZo%VcYR|!GqdrGaSYM@6!tPi5=8+jQ0?X=S!<6n8>IG%IfoSVP)+iIyE}0OV
zL#OjY&N85nN_kp=oNfWz(IPv-#upgBJ{OU4;_F=dwc&mburUsT9QqbC6?cSFjTzyv
z%CB#Y|A}=Rs%mBg{+PlsvEy#xZkh2XM$evI462Lh$eyse1hw{~e0&T4wBd3kckv%C
zS1zz9`v1S-@u}ZK{&Cu8R}Knq^WMH6f#C}R8wG)g%2(+@%U}(KxTyUy+*Z=FS4lk=
zb?**ykLamaRPWa^VQIU>Q4?(4T3rNzZWo~)OT8Y7<y-p#fHs0ekd&8nClGk=^WIb?
z5qrFJ{Q{ZPPxAN#_``c%g6kJ(oxo*pq4&PU@xFU3J@x9gK=`(wsPDBt+Xza}WZwM@
zdmV63Ln^CO{z|=@aN`>EjOc9*G5Z4Q@v1Bn*25uqKgP^*e(K}eiT`cgkh`#$itroy
zAjCrDr9eXkUg4Oqx;okXoBE|%iMbMM{Xlr=qn1jw@3eagE5kwb^M54o3%Ll*<#YtN
zQMdjR!bA{fw^~iqGWMq={2^f4#lrPLVzY^`^JKv9L2ZT+WsEZ%B<Wp`^ptISE#*c%
zzXk~k@r!U;*udYw{)Sdv2D;Ez1;q|7B9AWw$j~O!qE#W)#fLp^7#r_V;tJzxkvaWt
zsF1Z(oPl^{gzO`9MBOiZA)ftsc3kv+f&y17jOp{WuFbI7>7kGCc)wV}o*gOU?hhER
z(e;hOOy21cMu(hj6s!n6W}fl-mmI>BnFl_7G8jxc9K4%m$i-!JNfy+zSCpnLvqN|v
z^r}ug-6U6$6o=_{vfJ<8Lf`R_sc+OfBbe{eeOtqshfV7I9*=~W9=;ZRD6fYsPnqby
zR}XJCr@x#N+aB`HBR|wkyzU|Me>zscnt8<1=}rFuIw*BxNvD%N0_BihiwOxw>&*HM
z(|6#X19f8)w5gsWklxR1qhuSP%N*gbaM7YA>jPtCzGIFFrX(#6llP{P$EH<6!1<T!
zfsR;Exb26$l6VpuE9*`6N03Yg&*4VFQ>1Pl(WF?mIPz>W|1C;m-TO5E|M0&5doe|K
z0hYB?Bo||zvQ@*<jU9HHcZBUm3mav-&}^oN<DC-cH~;Ix+}vm55I$p6H6?UkTv_wA
z2F3>nx%VMxv09bfAU-V0h+DrSXw@`38f+0-wfmhLvJu{9Rtu&PwHMsiQX(k{YV279
zGpilv6LfwEfi`e0J?vc}8KF?I7RAL9KfC&<PNlcdN6<b5o{ieyps`uR$$sm>T*f<I
zf{Z15`}GkpDiJ*nQb$WyF|p0`*MN#}*zdxYLw45Klq26C!+QIu9dGn!md7Dq$HBgz
zX$1NVSFEpRr8dQk{;k`aHA>jHh~*Yi&y_`gmbPb|{tW6rmU=6Bgs$z#H;T6iR2At_
zOt1@hhWMG?uW6nNn>@z$NBw&Ai0HY(2*mj`fh>|I-2E1}AQ5<(ye7x!r5t~1JSU_!
z%FLE$U6}heuN)Qu7~OdX93UL$YFNCCVX{XNPHxLOuII_-Ij!pcDDe??PpG?5vGr}a
z%6Q#2oPJ2($TE!))I$4I?lq@zMSP}6m1APoLPoUThI>>n6L|`=G3`4QESZBoaC0i9
zkEDBnbw<10xh*3#HD!-bL%TL((x-A7w=~G~UHL{yAqT%ZDMfkK;kXs1qB!tvQn_B4
zr2?47!_e{p$TPm30^E)#6F1mK9qv?GexK_G<ia&g<?l9K+LzgMDtO~YC=;n&i{%L1
z8`VJc#q=%{vE5E@oMx-P7~<yRkbOY$-0u||1-Q9gmL&b8=;>)^N&SX%D_MT*-C|g5
z<3Yxh>-I@RlEit}K`fh62~am$p)0^-RuLhLBI48I6u<vdKStDPf_6XVG`_%dW*A{H
z&W*xlON<vlFdh<z&=vZ$>v8EZ)rsC5=zuJ)V)vC%bTO(_5x!GNZymYP+*p{5sW>zK
zVlCJxp~bz*xO4QZmCBM}3W3}ik~XGxW=O```o=|_z#wTcC8_#{8?#HF?xr&SvEk80
zTVo0$=TBE>qP$>PFp7f|V|OvNA4*<Ku`l*vOdqz(HvA}?4!^;OPm7aBGPWNQwN=={
z45`xt%`wFseS~a+Z`4i^XV<<*EkwH0Nu@}BDZN)`Ok%(@=OFj00pX+I<*kVHmSfPu
z9=~l?$8<urvNl@%M#&L9ENZk6btsg@$ya5FieVYso??g*b7<>?w3ZwpKv|QG$2#le
z0#T^!9A^=DM+-6Qr038^hQnJNAqZ%DseS)LcK0MO=Y-|S>S4Z{xtBm6G;FtoiJ}BL
zUKPOvt2UReL<{3^b%ja0`#+1zCHgaSvX%Gs>A_wVZ?a7$p9wWMMEs$~HC7wrzDZ3v
z$8GxL5%m46@nPF2Yi|T!uSBzCIb}}pczx1KX;rC?S?=o!U}p*cv+Mfs$`Px-VQP9o
zR~5l7<o618quv?iLmnSP&6c>|sVj8`B30$aS{T%wTB(I#%Ek^4C-J(7ysJ0_BjIj;
zcV4Gh7E;webyp7RYJe|vu+Xl6{Kz^XgPPC&pm`D!(wqCNRZ*%<s0e`GD2mF)wW!Kg
zg-vcZdHBSXf|bRp2z%X>y@s+R+A1LD*b)AR^|Re-8-We$)kn$3`pMz+Mu`NJaKHwv
zl{!fR<gzr5w!F9qcb5}<W|Fp9Jl?<&R188jZP(vMA&B`_R#!PILJ;n~K(0FLnMH)Z
zcBbwK^f(t63N>X6SB39r!Og9Dl<Np1L9ojW{0tPB6hpoWdj#bKn+b(>EHt`PP`qA!
z0XX8iGa1n~K%hi}`g3{&_@GPgGjS5jd1bCh>Yr;B4?-nZX`Azd3!hKB;!06^%~8*d
z)2{s02Q>T>WSP(UNwD#lB-Ef9^DflxS7f`H$92<jV$xFb=nvpuVYzAU7?+WUGqhrd
zpJl)(tO!_8Bz-hUv^Nrd@6@>v5wUj76}mTWU}44aeCJg;)(?w-VvzbVf_cHc;TP0=
z-DQuwZO-H0s9IT>2WvXLvBL-?wfjGwyZQRg#v2|vD=l6I*Mec(=YjEj<K5obEvtFX
zux?KL{K~R>D^G(WXWJXsY%N?mRv+Rz@1)8wM*y$+$(~W3^WqRmT>E?Wo@H0!+z8Fv
zMzK+>^=?fHn&#{Yi0%@Cj@;i(3z1_mZY+m{jWwyu+gKdC7n~oDVPG!p%&FlqN0zlX
zTE0O7qCQzpgKsz&nxK!~>gsl)iO5GDgg+cb?d3k`DF3g@JH<{vfE%r(MTCDg{`wa#
zn&gn)C{ho9aaLfgiir1%4sl})YeO<~m)>Ah3D!ajhY7EUD0Wg}+7cYS4>a<J@jk$F
z27qr)wBsEI_!atZ1J5G)<)e{#7u|rQl!GXxt~S{ZaIh~iwBH^w-?RG-RrD0ERJFNP
zIawyyI#nq2-QOvubUjIhCV7=UIW<g$?WqU%*^TV2c6R^tyS-9<uNeB0MCH`-*hh6G
z51Vq8Y}8RfQ;5e*>4n-T1}TKErjhGuA99;^K<_PeTRz@H2cmY`irh{ik-&%4x)V(1
z55C>CaRsXJkdvIK#fx%bnjOguFE<t1<xj_&By=U17#>g5?=THCmgHWsW&Y!+|3*Y1
zKs<1y2K_jFm@Kd(XDS&!mvbSSWgfwICDeJd`HXm_mh%b?R4+8snjEyG#x6NHmz<u%
z&rYvevH#;-u3mVpJ$bi)FF|NtRQV!+b@c2cVI2OsrO%)}oyI4kEy@uq^yr$Q2P|bJ
zkGIc+bV3;>x(UUr7~8e>!`CDM6H{vI-ZGV#&Ju4)%IZ)OhKKQA7nx!$NECo_Bu+Ux
zv2VLJU=X00S~qJ6VVyD7B=IyOc^D>OjtTYk9RnUos)y>@Rk4l{DQSty(;~GgxBu|T
zp8Q$>%S{6dlaOMRx>_9R18BuoE#kw)ha?~U`@y`F0G(M<mjah_P*}@Lg(tvHK6Z{o
zNzz80<4aro7t$HEwR}nRBp4GJ&%=NBNnrT&Q6FquY;LTTe<{j_-6aufwQ~|bOpx*{
z*_5N(CILZ6dj<o7emo=tak~_D7T0g|hdHYMt>neLdzXhm`;)|8pkT0BgaTFy+AO$r
zc|>z;ZOXA6HfQ{FKzXfD?}K%i8aao}#w5#~qa?l`ucoyg0;0%_c?^+FlHibc$}S=s
z!*&HzF|7DLT$l;$S7m6}l!Nl;-H8uNeo~!HMX@7Xhczr;^(6^xc2{RCG?s^_9n6yJ
zrx$FhmFiv|kG(}%cA(}EP!|_JzCCB{Hsu&U?6}s1J$$f1IRGJo{JTq@1*9GVN(9(i
z#(ro-S`rUi-BdyR+bfTgjVei@RTDaQg6hgOkp)sV<&7W8X%h&Qs-nUc-Q`CQQ8HD`
z{^C$fc1{QDJcPI1BgPcLG0#+kBkc*5N>Qe|7{K7S#XytS7!8$MY)9Uv9NJ;QjhRT2
zTK{F_<oIBqv}^MRj2osQx&1wuu&#%XJb(Ds55@n=H%lpMbBZbDb}-W_ydUHv@9no=
zc#X)cH}_nB<rzfLo{W*o>ay-3^u9La`ryj?QLH}ffZlrDV}9;ETq4^gRdKZ`Ukh9r
zq%r=`23c{lJa|`SbyjEtZC(&)4e<F~AC6I=&RhwbK!kCNL|%tDK4L~QY?yR<NUv8U
z@`6+T>yUtO)H*D`Jh)3rF_I8?-;0;u&*kv(G3bz~Vx01q&(qS)Y5(p>+xsJ08CjMw
zj96&m3T@9*Isi5zobu@pzkOFpkTK}tChPK9PY`x-`%jdq)M69fH}ByLdR$Yu9@Q=h
z+u?sUMS+N?L;xr6qhv*)ZC?Vh%DEnDUUvIOljhMxRXv6n2~XWJAx<c1RT+PSbgeS1
zg@2gQqO^;K@~;+)eveoLz?f@6v6oJ1tEo551+RRr!|j^}%CQ^4-%F2&=1xxlb~JRN
z)AhAP$S+~toDauZCZQ+))?wD!{i35m{yR1N>d3TL<KF%2?SGi%uBtZ3_O78egq7qg
z{FYmVjz-=o7Al;|X5B-Kr33lRpia#_af1+6_Lv%vX4%#*EwI@Ram!C1lB!rpY1M%=
z;L!6|@Tj;XYlD<BrXfYnMi7;~&<~RdM4^vP3<S~PB(>BshLF-j?i?bfT)!S^ia!v9
zb$N?J+4zS_k3~knp*_PTrzkI@xN2Fgdk6yrUP{tZv@y)$V<9LDydiD|pc|yB!t;^c
z*U`cp-NTZ0iPT!vO9=oCq)~0g|2wWPt@>Bk!rq||ntKE&s*M;uEHOBf31CEUjV|Qi
zORUIb=1Z5fU`ya_?aZ}1#Abkf&@814#@Mn1!Tsf&W**r(9XhI9JvgO5bf1<OGDR6z
zkYEAYbP|~2C4nD-b5ALA!7OWYzw}&NDI$kU?#(i)e}-(ey()Z`{VCLDB~ZY^4;6e)
z9~+}cng%_kIpV6INkj-HC!9_q$y&c;zuBDEeRyCqgCKIfq}~+FkY@}26Z??Zo()Av
z#ls|!^A!^Xj=BAcGOQzqj6!jU1%WiIylhR$Zp)t=RAfKT=R>?|kKISBp67iH8wnXi
zGBziPrB~%RSJ<r?(RMRi*f$XyLaOzZDWxLvq_|q_AvOz2dQV3CTo=S*8+%l%v)p9{
zjZnZE)VO&pRt2%Z_J|uJxEAETJkiQqv|0B$pG6KOXo*Zr!<OJ!-7FpG=^*yX>}N_K
zE+k12!3wou5dNt*i+Wf#dPMtfYUryw6C8SzLEg|0LlDA~+&gK6$U@fuv1B(i6WP#!
zbu0RQS@)pN+9kI=?ColY%ia+~eaxXez{moU5ZBW+S>fRaWiinl?DR{n6P+-kEe&>P
z%15Ciai2lO3C(2KCo#h9TvP?5RU$ei%}Z5+&N`To(9&o*rVe7hal@d2QZE~wKE!!{
zfG9jpNZ6A7*bg?cVy!kOFO7%`aa>j`ty<A3^o3WNkXuKnn1020&7plVaTJn#G^~XW
zA@)a~RdMW<yNCby+&Wh=0iWF8>F4pQxv(?ticeYCV09(!Faw@v$ngXdK$~E}R~O#-
z=<X_Vu*R~ipGwCZ{-0diG-x<c0kaqo=3)}SoE5>sZ(Z}-Z^5+F<OD7Ichs@DEQWSx
zfwNLzK6-IQweQ(luAx0$1+9!MFbE}!xbAJg35Ha0ypYsgR@x7kzJBgWsfwJXzIQ<_
ztm~GP>4B2@(Yve207!x;Ui)Gc<TOvHTxnN~s4Cc9q<jr(;xE%-ieuAhsDg-q`7ndh
z$bY(}gjNiHu%QUJkdVUvP$PwZbd)RhRh%-i=|0<g6MzpGznIM`ase_}ujueZrxu6a
z+0PKA95;4J;$WmAW|scXOk`?3ZgTENmsjz5GhQv`_H}(({_)6!gC`QDe1nLDh#Y-w
zMo{%(BjCDGV-DqpV1|6k2UU{`ri03^O)EjSS@Ax)D@Da7e6u~r)$@OWdn6kB9YZ-p
zxfL&7c7SJoQlmPm^>a%cnTUB*Qwc<D=r@d4+pERm{5|~Wz7@G~1eIt(vWVXN5*|!?
zkO0sKMucmB)TE;!oac&$Slta3K9CY>zeKQ+r#6y7Ld`OIPXfsg3;yw2fcK-zCXgSi
z6;eF8Z;K-HnaEHMWisEWK*^oC51{yZis|skPKMFTo5EksNfsqd{G$sK#WOzxr{_bC
zGZ*(GVnD#SLrXwIW<j6bLKPQU4Zc5<u*nE<S|TyP5)38h%Ll*u?4yH-V&&)WKZziQ
zKkhI3FdSxW3^|#tEl}~I)TT`oSO6%RK2$`!ui#$&_E{O*sKbr)-%(VE(vnSdUnd9j
zZ#~zO>@(*}E4JNs70>+Ga1-O6j<V7!B$Gvs^#^hX8>A#V(Ms1Bim-eMF%lSfq5>)>
zI38xHeNG@-DfGQ?#4R;6Q(U$VDO@T3bk7Z$g5G{F_=8f!EX?tL9a7NR`+N-fDx^HY
z(Ux3-%C)Kp&Y8un&v|F3^NC_#JkvU8u$0fLK4OW;1Qa_W!MMaSxLLSdZ|mMZg4#k9
zGM`(vsCeouYh`#%KB$zT&0vbp;x2;M4MJA(;U*~3np@aJ1-s-{fPUy6K}3pzLe|67
z;}D^W4%}L;ENmB3|F2R`y2p@GU`VDWvV>}7WpHM~ink?_Ga`9u8YPGIa~9SJ#Gi;~
zKWU#-nZf|m92#Mu@?lVXXg)z0T&wb-A0J=?DH(CQ?3n!D#j|%!#}r^Bs)|iw4GZ?q
zC4JPB6q*!Of!zHjxUfYtC8C`N(1x2%HdhVy)@;#om9%v_Kp1s)8xK){`evHBfR)ct
z)a3pOMMWetiqN0)^p(rHABV!>05>p)B#T)LJSWuVsl5pJ21UKd-$O4-Os3!@(=Z$a
zdtysai~}M!jCy7L_ts(qY1wrm((ihE8CMER*1I!vlbb*G8Mmv+vV+#N?>MupqJ=QV
zgZngxHqE{h#SVY~-~q&0(k>*Tj7b&k33PVt1?-+;95`ix&BhVPIw}@;RP+(XP%N|!
zH(RO|3em0IZ)KNlmMD}mJPF1r+sPXP#1qSi1*lgj*6NX7rpoAmp-G3$BE2`?5Ohh0
zY$(|7j-TF_zPgQHPrJZDJcm^!sD7mY3Hot%hI9#-O_iVbmT(dJiVX43LP?Ggf?Qht
z@YgR3`!K8N7Tq86*ktPDf|XlKh<&`?{-$fa-CAp(!dDqepHY4#D|hdHQZTifCi!1C
zP<laU#7~`K=NFUYC{^Tri96M25@`1l&(f=1&8Dx(|18(_^9vuVLE7GSRo|Kn<9s{R
NxS@3Hm>&}Os3O%?TsHs!

literal 0
HcmV?d00001

diff --git a/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin b/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin
new file mode 100644
index 0000000000000000000000000000000000000000..97bdbcae01d8838c3ee046f227b91b99893daabe
GIT binary patch
literal 8047
zcmV-#ACTZEwJ-eySQYyK`m2Z<EZ~ktO_Iw94g*~r(Iu#%2n1?()O_+gfQ&L%_WmiQ
zPeq?nfMFrV7#kxd1U~{@0a*bM3pj9qIT{6H8bza6U<_s;fN2aS5^>riV$moT92z_%
zjb-wPIGDz0$^#6}5>XnZX%w0S38$gK!b~Pkn^+oiz+qbRU~nFb37FO_2_$eD8ZeDz
zY0YF&BAi3X(i#fGw9_1>DUQ?79*<VRA<@8T5fP>>mIZ^uctjkM#yB7#PQwy$Gz(@C
zA;M{B&?ZbEoYq(tj~1Z-X%GxfV~Id`s6euKaF_$(;lXJft&tq1MJyte#?l(5J#d!E
z#93O?D3YZ$iy~<(%Azb7h%=cut?_V3NSKHN6O5%T7F0M5jb*}ElnACda2Dk(ZDLt8
z62-x3OJg*}At7Pf1czx6XK9UP(nyr{m_Pxtkf3SW1E*0S%|z2Ar!^9$ou)WA$DxrZ
zr&%2B(HhI*G>BudOqeVtOo%iRrcD+Noc35i6bcrQWpSFqIH#cv&`=%|k_7`iAd|=f
zrm-}KM3Zn@!#D<bKqiv~6Ix?Y+JQKfvqUC1ZBg23B+7wUz<~qjI4m9$2oo{jG!|u9
zCK65n+aQ@NPJ@PNjnXPG?J$<fX_%%=C=(P$1acaOgwh@i4r5UyN757rkoKTha0q~c
zGSP_OG!}&skyzSkmewp0iQ{OL3Ww2}MuA9{1=F0RwTL5GI8L)P$V4K-I1<GHvA`fr
z!6+ye7FY$0NQ7eHL9y_lkUW~QEF2sOBo+!Bhs0tzOM5(8!~%k~N24@~a~2q5(Hx2e
z_JA}F#R7v60Rn;ZfB^^~3Ru7a0E%BLTva3bxpJLs%*E$b;pW|<`mRQ&b$vxly6E3g
z&ugTJXmptBwIZf}y+zeECPhr6M7OBxOp1C(nn@FllCG^Z5&u(quhL8-wHKGJk#U!1
z`cCg|?~3YNbgEv&uanpJL;ot@`f9Y$-{rNhsCebYe|@<Ab=B%{G2OVvf7M@f^)+Jt
z=-ZSM*Vl<_{b?ejLdDhFn-D_0#pC)%%l<+=gp2O$Q+0ofxYk8acqgT+s=S!#7IB@P
zn-C>_RW%p$t8Q`OT0J3h(xcq-Uao(*Di@{8bgR-{^y1gK&Ry44sEeHR=RIBgruQ~;
zeYfz7e5(-Ei@2KWTik`}^0=yt$&3Ge7vd&+m2;6UE-&7uT;r|1m&tem^s6*^jf=)r
zxpec^*1h~DU#>#9ZyprqaLE{$)+|kefq<o<JQj!1D9$;DF^%FuvA~$-;mfF))){^N
z<?qgp=0On<kBG#H&?FWN90(5&n1cZs2v`;rBo!jeaXdUA9#4&qKSfl(`|nJ3_PQ1E
z>gQY0n796X^sO~Hoj#v;is_`(EmFMybftyrrym;c?pB4qDk3~8RR2pCDs;Ng>RwcL
zUCNkuG%jD}5BD@tF&)x8I_f3jIiHv`ofgqbab2!kE5-k~=)4sws`FMqdTA<!xJ%Q|
zMeiN+UPcJ_j){Iu-zyi<PvmvApNV+mbJ2@Wuat|%Ya&D{kGymdd7V#mD`q~Cm*V}I
zNA&u+6rHAZUHqRbd7p{b#Y35_OBapk`26BC;r*D4xPIo<Q{sQLy2{gp);-?U7j=C+
ze5S6~SLT14cegTqd3wK1#l1|GNY{6|{@YxbUbm@qt$BO8eD|hvIrVv8FY8$kz0zN+
zTcg_d<`XgV-pv2jcxZol^SQTHKa(DAKKIt?MBGGQXS!6Qk>X|w3+w@D91jnQg@q$h
zM@5v@BYL_zDqcc*PX7@hgox6AT*rJm@AQv~6xZ>Ho<E{5MCpkkL=2NUN|cb*rF}<<
zGVur{{)rM%a!8bjR60tQ^3Dh|A(WSid6X%sqV(}UN|ni1PU<MrS|~Fmgz{1<hB7am
zijbya3Q<yDNbl+F#1v&hKRGjHQeRRmC4I_-)H$uhRH08Xq)M@rifMXEg_25@?kJ&R
zizQ`c>+Mv`>uF`GV(TkqX0Ahut>39rt4!)kuT)_wwb)fEmR4pSyYE+pnzz`xLgvNv
zez<&<xKia}N~tb?DfXr6>Xa>A$`H!?Lb{f&#gKzu>UA+?m;MuDM@y+{QrD&Ixz<v~
z(wB^Uh^J@jD@#`UcRKUv7)xc$dS$-qm49_*Y_UZ8dpaZ|>d`$BOAhLYeI2144jt1o
zV(Nel6GP~bXvH5xjImn|%J8Th&R8PE7-K9Ma!80V24aaZ#-ceCLNiCBP^M8xU<{c6
zW6+A!s8sN@wcyMk0099&0098g6951h3=D`&Ljo#o*Z~xP0U8tr5ELv76bJ$YKycs)
z6R{{100u%~Fc2snR5S%n0%!pBBOPGdQ;==LD-nJdO=c4|$1C}PGP^V5qHDG|4zGZh
zGuoa6mpUIw5*KB<y|$4WJy#KHMp@tHSsbU2CFLmB^x@&XWHrmQ<S}@;Lr-MdA`9K-
zKP^?o;v#3CHBxb-#!Iqdr-3HbAyS$MsZ#>$2&)LTzs6=^=~3KbQ!DjV8nIu?cXB|a
zEQ>?>3gr>1_ax;Pa}Z4|B4ke5@^yrrZ7847!w!y0bF*K2HhgkThvLXtO-T1~R3KwN
zbg&IoG}3s-AjgxqUydGO{3V6r7}qdy1gMGYM(7M15&82CbX-9nG({<+pM%j~9B-KB
zA7SrN;DeP%1k~n7H*D=gSo*N+gJzzpi!at8g7+ae9)X{_cQ_w^IFG>6FS=Gv|FD4H
z2%<O|Fc=oM04+xH5z_Mhp}-SSep_c<7ae)ln~=dsBkTvR^x7Irpz{^6@`x}>82T2t
z8JQJ_s^P=o*v64g>*6U`ja<d}aJ18N&Mkx&bSN;ww7|K0r985$Ukvlu3hZYZfNnwe
zGcnE~aHCR0^r)|zQ^MX?TjmjgGXj~_pTZR97Q_X~Gb$0{!`moh?=D#ZoI~^TW56;n
zib@AtfqHJibkS-#Ldh4{zv?Ffe?r(0XV%8-dsfCcV34;y+yS+Y@a$tE99{YC)*$!9
z_=jrdjQDs=IVX1REwnAO_KD52pBMwr5e%{?YnQCneuIxsTpLAXQ)DUr!{y4w9tG|H
zB|g5{dxk%D^z7jx@&?3Pj7MmFfor4Q5kdPZC}@?bp-Wt}U*6jqbM_LcXQHm8rQQ+!
z^@?cyiYM%9muOZJg%Idv5fYEouO4Cc3&7h@il87bp`1XN-bVr*P)Qh$*P&l6A@$V-
z<whUc%S#IV8mSWm>}~PhuZ|aQkJ+aFY76dfyNx<s>#&Wm+|0$f9}2I-MkP)~oys1m
zZxWugw@wlJtRZz@ARO<|WdeFQF7GFjSvE?2eLK0|rX6zg7Jd;6hdxcQ0Db9dsGvI>
znO4`eHtVHcs8)$QOD!Lcg+A_5*%xT}p90Hp4FCL1l6P=64=n<D1m39E|D>rA)U%VV
z=35zSR1(z?&~mYeUqH09BMhF5`1d@UVW*675)Kx<>wE-4H=PRQ%8Zo55)>|r=qPO5
zZ!q$P)-SW0(EgqO9o&WE6+eK1aw4vN!!X`69@m1ag<|@DLj{_pwuSjCAjl)=ME$l=
zghyPx5P4q&`~-2rDnW`;sTL9bf{Hxsw<BS=`xurqGH9b*mNGj+(2#K(Wi&#inWtO7
zyh6y7qzah+5DbzJN6x0hxp*2~;RQus%baOT>JU~3eZ3QVJITL<#^G^0yZ76#&}T8^
z@s09-1l9L&Z|gSmL=;87hag+x`<~f9H0dGRQ{KhzyTTjh>EGwXnjQ{&WY`a9YU0Tc
z8UE>IJc8OIj7}f*N5MhC8;d+0{s_oJKJj8*I1YG=)0uu%_Z%8D*3_odj*xIaM>ncP
zU%mE-(ZVGjEm<Cz+IdCfn6;9GGE9b>h6oJF69W3bD2Z&uw!$Sj<Za^lYOHu~$3B9n
zGI*0ViUL7Rz!4$EQm>I`v-uAI8q3xbxBmyV{f`q<ZWkK2B~x-y?<pHKJkeOM)AS?Q
zXC%5&;yF+SiZH!X===t;uG!{>%s5QQ#nm;HbzfAg$!m>{kE}TSu(#Njl|&IAz+i}5
z#3Q)Z6gV2@BLKB~)D2S+UO?8((^$<5Ue!|Al-xG95Ch|{9XSxPeK<rL$X0sYuBap8
zX|d~yi;wK=V^JOF-UN?Oe+cxALTIVeTSWH#I>DUOIbH%8TlR+XBT!c&`VInXOG;y6
z^7LA95V5o0ST4Wstbvpx-5>4u=36@y(Vs1k<15DjydRgR<ms;1uV;-pMWg<0*&AM3
zYCaL)76Q*@e0`Q_&w9Kwr2kmyP16x7w&QG+P!X3DD1SqkU69L-pWgc=^HjdRr}Rf_
z3H^@fox+IYe41b#$rJ8=3tQ0EepybFyY^D1pBmr^v5hKaOLY_pzpqx_MVyvtJ_j5*
z96w%Ci`Hc9a)jU8)H^OpvT;wVKY!Ht5vWh7H%h9$?eia*+iFjLYTf9U##C#ucPciT
z7g$6jRHUYuK5Ky^THgjA71Tt;g26FKo(f63>mTTC3e-pRleQMPx;@>dEj8t^N06a?
zn6c1PIgT497foJ?Hc6#`-%lxpoV7b{QB$`b@W)}yI0{Tr3t%Z8hL%u3o^kC2OZz_g
zxXC=~aHnGW{dB2z7A|2bL15Dzzs#Jc0vxyTWpeFW!Xwb$DB95?X7~4q1?|M)H01u(
zAGh>|Yz2zBey`i8T$$UgBvCW2=W{zx^<yo!_VOdoVt~FnrksDZh`lQ!KO8Fo(xa8m
z0_;k06Tb%{;EQ;a?lb7e#`~P0!^fQI7Z7I#Ff2y1QPz&2<YIOoq%<k$`O9uhb%$^6
z>44%_0pNzVqg&IW2oF-xwhrNL-gB^wi6k?uVqM*+)n-^FpCWm#n+ixUor|2$&oZXb
z%&?Aiv5kxGf!Vr%O;T(Qw**L@>!t$yvES(0t1%govsYKwKzWha<it=ZHfS*=AxmBi
znO|(%m_QzrZFo#Jp&O|PhZ;MN)F>aGwUv2c22ayt#WB^7zJ+Wu*(e`!v*W)9WQKId
zO@&B)?Y<XpO#H$#=b-nZ5aOfc;;o3VmSfk#u)eJr$Mh~+Q5fxcqkjB3EKIZ*a;U$>
ziMq<Dj3Z<SY3BS0Lug|stsY0nRMsBF<8JH108!ZOd~XqaN86tTfHSn2hXbRI5Fcn`
zB>TP~yYlOs^Uvj}Me0h;2%5hSPqtgOM3LCjsLEdgs5W=3#Jh12jbfxd^PeROauGXo
z_Li6M>0wfpaa=@9i!3!lQ2v448cPG?9#T`Bf3yDN(S1MY@zHFQuN$F!R~BP#Ws#ir
zcv|TdpsLhYQSOBbV2qXJ*@gR{+Y#K~U@<*%Rh8;4AbUl(QKLTOgE>Bq(3V8ssaSPJ
z@T$stEmhXt-cy9ZjN%UGCsDh2Vpq8YBi3#Ys$b`^tZS-&S1U(=PpCGNR%ik>3}>1$
zL$?0@a6bvw(<9s<Rk^&)b_CGbsK<rHl|^L-q97+Sd76kR!>4IgpuKK%U&Fj4mK8we
z*v9Y&@Yy1?&9@DU>_;PF{rPabjS3wP8o~xS;+jODf*^LAoNY^l`^pK=Y>#cak5};#
zc7jmOZ5IGSNdA#;rTQvGBLsWze{+Re&)|p<v@;lw@QicXLjf#fyp6u2ElX~#7qRXb
z2{yZ&A!p#Aq{#lt^bsN_csdlOVWH_rL2>x%1KJV$&eTcU$Uq6I>QCzt9D=Sw&$)V#
z$(5FWUg+s)@l2JR(A$6=95B9OoZvCNM%Oc}3|AKE!;+B+GM=ycgrRqwKhz*N=Ed9Y
zR+QV!IJ&7BVBYem@Q3BU!m4fF8JCoYL%bpu?rOj&t%wOk%lbA*wD%~7Xq3oui12;$
z%H50GU~xrw+<B@TYr`X2F-UCPz@&IT(B;xx9r8y$H|N)HX|0rt!91PI+QEhLGJ7A7
zQ1f(_k2gJYR$RRBttG>_$OH5G#=O1r8$_{uW}Teq>y^d&R-P6`&bBv>{aQA3oOt&2
z;z^m+jzE6%>z-jbXAU8fu=dw9Fi)?9fe}J&8#ANi*1I_=*q-z8Uv!sN9o6}6K_Wi`
z;;~#UEytv;{YJsqH8|fW!@zIa_gjPi91&@uxBL$PLPR9VG04GO?u?m4y{=WMCc+5h
z0WZVRZLcJQ4*&eBsZ*r$1KbFmXhb+?<KMsF5+pn6{WJBBL#9lRxDk=R=<*w*o;Avt
z`}qdro#2^hv6oy%IHHsE(`?ab-+@*?j8g-AH2}nO!o1#L)?Fd~H*hSHG_f9=chSWm
zrC5kk0Ffs7fdhRKr~USz?Vj0hsM1DhrAmpnUx7_=Ez^a1m-wAxO0OlUE+lES)24)}
zs0H;vzrE<xy0CR}bbVh2ur!ILm#}imVfIn3g@;W!KsL%<KvT}bOu1@plxq}rdD946
z;SbrT9VUCrl`S9m&`-LPu_7l_;eX)cVciLF^G9uVZEg;#amcAY6_Q1{Fino6q?eKk
z>+*+VH4-|mObq4|op;axBTMqWcr)*@Q1?w_A&GblrN-tsp{*>izh`P0zL#Smnq?lr
zc_rj|v*C?6r<UVNZB#8Z+nOA-<i;*BIG3EB1I|ugt)Ty79Ijq?jXim{053sm8bSGL
zymb^_lCT^;jHM6Bo<6`QaxKb37p8jMp~t4vOP=^=;wDZSCf0<a+k7)x!^8iQ0M?Wm
zy0;W7na+q~sdjaA$>E0hQbneK3z8N<*@#oV8#Xned#K@uxTEbkwWsULZ%1T>7qwG1
zQc0DOA6j>sV<LQgM;wo++e1a>s+PwHIxS#%+U;%1xpxSIPrenvvWOwWB>Nbp0xb@A
zi!i~G#eTT%km92cJ{VLbpct0aOyG(Q3fKQqaSfQq$M#6TjO3#Z_R`kpg#=pK+O8yu
z*u;8`_Arg{NgO_X-VbI(ls8tTUs~F*qDX|Jb|AzLtE4<|Hs$FEmw*f+oB<T08;?jK
zZX*~n1nal%p^x=@FL^ufj`FZ*e{QrZP`cS{S-=X3Hb`#W9nqW`n{qFQ%o#r$P~I!*
zeXuK2GxxC2G1t0t3dHwD)v(q>Kopr_9>cMdI6Taqa+Z+|v0Z^QhA+Plnq~q&ptiL(
z<*mu_?uSE`pR{^Y#o5utA)V!Ow<ICP?$gYd8ug)=hqJ`?(*c_br5cvUGqWhe6R0}`
z)I{P(wCAkdrX1skovtxqAscK6E<mVY|IWH+dDTNii7<htu^$?Zmek`^H<f^Y3+3Y+
z#+96E)yU4BLS3<5WI>Egxrv8|+XRZGDypy@cDd8TxJ(7Vzkn5!8aLfq4<WyMf}bMf
z7ChBdk#?|Bk(Q~N1~3@5_#WpaPDAMu+sWIMqjuPEV=9oO(tQ~!IX;+d?V9}oW5YCb
zY=4X=#(rqX^Mh{vr}Mw^XDLTzPAQ?>f@WHU`iFe%y#4eGy%U)==bi|zJVO-mkumT|
zT=qX~|7)aN9~{LW#czjr=&f=+W~T<1Q6l>#Rk5`xXH`xa%$e&j16i?H9+<0M$RlXO
z&_!=87O<be9@=+CommmKeT2b^#GXT~KjN%rd^G9QNUs?r6v8P_`XTx`YLd;rJTjNu
z97*K7Z{5pjXMgyc7_gA351jI2dC=0P(|UCz)cPZU8rk+323KgpukGR~WdN*yIOYEj
z>3Nq9$oS>qhW7F+(-Q2m`%lJHdS7MsG}^=TcwAFmgV$ZM<so4<y{w2WB0!1nV`@bm
zwl5K}%IO{|X?9aU*7G1oRk4H^R?TXe_f9Cjs{Fb^l3Qhz%07(OqJ2ffnyprEeorX^
zAi=e$vzKt%I-xgJgjeq6kUXOY%7z<J-AhG>5U05SJ8V61i2C{$@>xd0sE4jHllVZs
z;34hodJDxMFH!Y5wpluqaXb5~Fvu~>`%RF}_714cS(RiNe#b_k_#$tTg~H;u8|_2C
ztpibDP&eH@(IE)yYD|lRY1wU;&TMvSxaBAhy9q4Bw(2vze0cn^K4>nv_8`R=Y1E#x
zA`qoe`9t>uA@ZX;0znfvNr?OzLlb-OZ4S|r>*Gc`><=(&UC#d@y#68XvE+z8+}LoY
zDN@KNaZ*d8Jxr&8mr7d7Z;ZP7SeD5G{)!tK6oYhCc!sh2Kv|gEa`;><vEf#ADS6RA
z{#u&{f5%r#7yk-A>^a^U<1hhAU?T#DB7>uC0x%1%zCwok5|#0rd`V49lM+~<ojGEM
zJptG+&650JtjkLfuD_gU=9pV2Lr3iDAyVsy&S(iZ1-)@>F&3aLCxIzm8t@VN_rx|A
z)M9PkAMe_lHgafkZ=O*-Gh}O(RlzecPnB&}+%?8}DB@%K^o+v7G=va%5?6>OAxbd8
z;&e78YyTw!WOLrwp$yCn36X1*`iL-tFk7HZ>;q(buf0sxAIf>D1e+)Y=C*~xu!bBe
z3`IZ|1czaTvo-a*EiG=)kH|dV4kAu_EFP_7p2}+wE@YU>*tH~<LzOf0<u`If*T!rK
zV0*FQtk%<EEfuLVMCy7EF%pz4QcT+Cvm)NZ*h5lh^%x8iS%4LAck!5<CJLd^BYKqJ
zBFgjfL~U=`X5Hv~79C1}B{G?WErzk0NjlKmMbsg)FH-_^Au&ZH7iy0|_@mw|&d0*O
zi1pm8p`~{w8TF>JykQ@P0F1}6cfg3P3*7-C$!_RmvZDd&R<!?T6||qxOKxe{8}*L6
z-4Uxhl|u`F(RMTtCrsBp!vhw|g10&G>92Sv`h<vkHtkTEjY4<gK8uPmo5^LL2Rv@)
zF)9h0MqC=0H`)a4?O<Bs%A?sabpZPY4}%^?>DcK15#j#=QFzmQVN(&MADm#th&Oj#
z8sR9!VJFzwyJFM#MXogXx9*~thKe>cC)H#sDI^GJSTY|M#g9&?;xjJk9{BIQb+2MN
zKDjm1|M5ZP(AH8FowTss>q^_o3~<Vj;|mrDZO(#^F0Av>+g0>X9n1cCDjIXxe{yMS
z(Bno0WHAuVMI>PJsCb*-y6(5zhH2-MGv))>KI(KXA2E^r#o2+-kB*LF&pBJp%8)Ma
zgVqEKES{3(s(Y$8;U=%B!16tl{b?AcO`dybs$%(4PhC)1qPR=?q8YyV(Sa)l{BjWO
z>ApBBDa{ivSJ@R4s|tG-310)6hn8s}0oruht1IH5J<Oo9_n%@Z0jI(LY$%>AB((6a
z)JP#89c#tL4yUl(-gmCv#PR{OA+uQsZUoo16|GjH^Wh;3_E(^k+>PCzIFeM<nWf_)
z6WLman;hq(|50QvjVFV-ah1L(TRaNm;E6;jWF=xFB1K<YBPel_l8kQT%%QF$nA4l`
z;i}1v>EN=9#7bCZR)inDx1xjv-;l|1r};lj9tjTp238If+)5cQH^DPvOxzmP%HI;R
zO++NsOj;2q`Hh0B&3>^be@j0)AVqC;1R5=fAfhI@gbFhbl4mr66ye$hoD|TIaIUZl
z@a`rf9?03+uLo-6Ig4bFsaezClMv-62LFy00Q~65Nan|`FcjbJ+oDL^Oly`S8O3f?
zu*`Pu0w{VdiZOX4t;5LLO{Z4pC3BS~?xUNDBAFk+sevK4lZ!h!F#u@XaZEravw({x
zfr`nl1}~J!*<_p$EwLD2D25XA<$+%P|ItH0u>$kgpG1)2AFo$^_!?$R4E@Yj5~>(e
zs?#P4Jb)LPu9PCOYr#4F?pYZx)bU1o-cdw}(#a-@yVFGUXFb<b<OAo6Q<G%7ig$io
zxQTY3&T?r53X^%px&U$e8B`~`c%{z^#aq6Fn+nVwQNhs@Bok)nK1Yx(M5<OiqAWF#
zDNag<P_7hjn&$#cLCbz?SV1UaE}pS`9d24}n0$<#DkMC?*Opx2>TOYR-rVBA$a&7B
z<H}-Qo>2ufK$VZ8K4e8C0*V|)f^mT`c=@>OM{AxRA#EWFnXW5YRWwJIu`s+$Xi%Nd
zMqrA|<6eqa3_@1;FjEvIlUvxM0<CiE#eQfWF${`=Hr7GZ!-`PFC2px!mUoKj>#q<e
zy$2ypGGtdDa*S#wWGH6BPTmq7F`~%QG$jY==V%py{f~%cZO|?W355xzIkLh)(P1U|
zpsa!_xR&k1JPyDHQ!-+`?122=wP*X9z9)cqs)|j$8dmI|bNZ+|C^W9B;#hZ0@DW91
zLc}-^q&AsOb}kl7jkC+<%4q9#lrn1XH6G#s^-Z>OLAA{xpb1@!oQX(g6rnR`X->_&
zAIFWu;cj3INrtr8e@-YTsYems4T^D*e~VtM8&JVX7Q>(k@Wdsds73c~R(fU6Zy$^O
zwq>_SWccvFGR_J1y?1BmCO3ZU8E&_nWmkNo9p#L!ilV|u)_122ttVV36k-6_hlgQ~
z#$A3fGCGZPH{RJV1uWf=@sr6;m>BU-)@Cf&R54^2X*p@*(rjNVC@ifV-IDFnY&DSM
zPW6mg-^uF^l&6m22C%n~o7JP<OqI?7D@mNqBE2`??{rCrY$#amj-TF_zPgQHPrJZD
zJcm^!sD7mY3Hotn4Cw}#P1Qc{&2SO-iVU%z1xSJrrCeJ5@YgR3`!K8N7Turl*ktPD
zf|XlKh<&`?{-$fa-CAp(!dD4OpHY4#D|hdHQgFJPCi!1CP<laU#7~`K=M$6USgMGL
x5_hW4B+%|9o~2j2noVDm|5>i<=NCR!1Khpss=hTD#`$)reM9NkF+URcs3JH;VtoJr

literal 0
HcmV?d00001

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index a806f55ce..63a69f7af 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -20,6 +20,12 @@ FORMAT = {
     'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
     'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
     'br': ('html-br.bin', 'br'),
+    # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin
+    'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'),
+    # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin
+    'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'),
+    # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin
+    'zstd-streaming-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'),
 }
 
 
@@ -80,6 +86,26 @@ class HttpCompressionTest(TestCase):
         assert newresponse.body.startswith(b"<!DOCTYPE")
         assert 'Content-Encoding' not in newresponse.headers
 
+    def test_process_response_zstd(self):
+        try:
+            import zstandard  # noqa: F401
+        except ImportError:
+            raise SkipTest("no zstd support (zstandard)")
+        raw_content = None
+        for check_key in FORMAT:
+            if not check_key.startswith('zstd-'):
+                continue
+            response = self._getresponse(check_key)
+            request = response.request
+            self.assertEqual(response.headers['Content-Encoding'], b'zstd')
+            newresponse = self.mw.process_response(request, response, self.spider)
+            if raw_content is None:
+                raw_content = newresponse.body
+            assert raw_content == newresponse.body
+            assert newresponse is not response
+            assert newresponse.body.startswith(b"<!DOCTYPE")
+            assert 'Content-Encoding' not in newresponse.headers
+
     def test_process_response_rawdeflate(self):
         response = self._getresponse('rawdeflate')
         request = response.request

From 6032a9a31065a88886ef0f600c57ba63500b95ac Mon Sep 17 00:00:00 2001
From: "P. Chen" <pengyu@libstarrify.so>
Date: Mon, 5 Oct 2020 23:55:48 +0100
Subject: [PATCH 161/716] Minor adjustment to the test case in
 tests/test_downloadermiddleware_httpcompression.py

---
 tests/test_downloadermiddleware_httpcompression.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 63a69f7af..7e9f9cc5d 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -101,7 +101,8 @@ class HttpCompressionTest(TestCase):
             newresponse = self.mw.process_response(request, response, self.spider)
             if raw_content is None:
                 raw_content = newresponse.body
-            assert raw_content == newresponse.body
+            else:
+                assert raw_content == newresponse.body
             assert newresponse is not response
             assert newresponse.body.startswith(b"<!DOCTYPE")
             assert 'Content-Encoding' not in newresponse.headers

From 371bb808689bdb6e2bfd1027069e85716c153b51 Mon Sep 17 00:00:00 2001
From: dswij <dharmasw@outlook.com>
Date: Tue, 6 Oct 2020 19:44:48 +0700
Subject: [PATCH 162/716] Explicitly declare PyDispatcher as dependencies

---
 .travis.yml | 1 +
 tox.ini     | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b883c5b78..fb4802070 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -48,6 +48,7 @@ install:
         tar -jxf ${PYPY_VERSION}.tar.bz2
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
+        $HOME/virtualenvs/$PYPY_VERSION/bin/pypy -m ensurepip --default-pip
       fi
   - pip install -U tox twine wheel codecov
 
diff --git a/tox.ini b/tox.ini
index 8cc522378..10b144619 100644
--- a/tox.ini
+++ b/tox.ini
@@ -77,7 +77,7 @@ deps =
     pyOpenSSL==18.0.0
     queuelib==1.4.2
     service_identity==16.0.0
-    Twisted==17.9.0
+    Twisted==20.3.0
     w3lib==1.17.0
     zope.interface==4.1.3
     -rtests/requirements-py3.txt
@@ -115,6 +115,9 @@ deps = {[testenv:pinned]deps}
 
 [testenv:pypy3]
 basepython = pypy3
+deps = 
+    {[testenv]deps}
+    PyDispatcher==2.0.5
 commands =
     py.test {posargs:--durations=10 docs scrapy tests}
 
@@ -124,7 +127,6 @@ commands = {[testenv:pypy3]commands}
 deps =
     {[pinned]deps}
     lxml==4.0.0
-    PyPyDispatcher==2.1.0
 
 [docs]
 changedir = docs

From ce6884d517abb75b884ae23ab59e1405daa83187 Mon Sep 17 00:00:00 2001
From: dswij <44697459+dswij@users.noreply.github.com>
Date: Tue, 6 Oct 2020 19:51:42 +0700
Subject: [PATCH 163/716] Update tox.ini

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 10b144619..4e6ca3ec9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -79,7 +79,7 @@ deps =
     service_identity==16.0.0
     Twisted==20.3.0
     w3lib==1.17.0
-    zope.interface==4.1.3
+    zope.interface==5.1.2
     -rtests/requirements-py3.txt
     # Extras
     botocore==1.4.87

From 137c8ba6ee393d0887373c88a9f252af4efd8e3c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 6 Oct 2020 10:50:17 -0300
Subject: [PATCH 164/716] Docs: mention limitation about Cookie header

---
 docs/topics/downloader-middleware.rst |  5 +++++
 docs/topics/request-response.rst      | 12 ++++++++++++
 docs/topics/settings.rst              |  5 +++++
 3 files changed, 22 insertions(+)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 06e614941..ae84b54fb 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -207,6 +207,11 @@ CookiesMiddleware
       a warning. Refer to :ref:`topics-logging-advanced-customization`
       to customize the logging behaviour.
 
+   .. caution:: Cookies set via the ``Cookie`` header are not considered by the
+      :ref:`cookies-mw`. If you need to set cookies for a request, use the
+      :class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
+      current limitation that is being worked on.
+
 The following settings can be used to configure the cookie middleware:
 
 * :setting:`COOKIES_ENABLED`
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 30b1945d0..f3aaa2c8f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -61,6 +61,12 @@ Request objects
     :param headers: the headers of this request. The dict values can be strings
        (for single valued headers) or lists (for multi-valued headers). If
        ``None`` is passed as value, the HTTP header will not be sent at all.
+
+        .. caution:: Cookies set via the ``Cookie`` header are not considered by the
+            :ref:`cookies-mw`. If you need to set cookies for a request, use the
+            :class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
+            current limitation that is being worked on.
+
     :type headers: dict
 
     :param cookies: the request cookies. These can be sent in two forms.
@@ -102,6 +108,12 @@ Request objects
             )
 
         For more info see :ref:`cookies-mw`.
+
+        .. caution:: Cookies set via the ``Cookie`` header are not considered by the
+            :ref:`cookies-mw`. If you need to set cookies for a request, use the
+            :class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
+            current limitation that is being worked on.
+
     :type cookies: dict or list
 
     :param encoding: the encoding of this request (defaults to ``'utf-8'``).
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 06234c5d9..71331c841 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -352,6 +352,11 @@ Default::
 The default headers used for Scrapy HTTP Requests. They're populated in the
 :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
 
+.. caution:: Cookies set via the ``Cookie`` header are not considered by the
+    :ref:`cookies-mw`. If you need to set cookies for a request, use the
+    :class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
+    current limitation that is being worked on.
+
 .. setting:: DEPTH_LIMIT
 
 DEPTH_LIMIT

From 9b1f86b613d2039b0a66ba2b527a7e9bffadaf3a Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Tue, 6 Oct 2020 18:50:55 +0500
Subject: [PATCH 165/716] Use f-strings

---
 scrapy/utils/iterators.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index e140e3f6f..3b504e56a 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -35,7 +35,7 @@ def xmliter(obj, nodename):
     namespaces = {}
     if header_end:
         for tagname in reversed(re.findall(END_TAG_RE, header_end)):
-            tag = re.search(r'<\s*%s.*?xmlns[:=][^>]*>' % tagname, text[:header_end_idx[1]], re.S)
+            tag = re.search(fr'<\s*{tagname}.*?xmlns[:=][^>]*>', text[:header_end_idx[1]], re.S)
             if tag:
                 namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group()))
 
@@ -45,7 +45,7 @@ def xmliter(obj, nodename):
             document_header
             + match.group().replace(
                 nodename,
-                '%s %s' % (nodename, ' '.join(namespaces.values())),
+                f'{nodename} {" ".join(namespaces.values())}',
                 1
             )
             + header_end
@@ -56,7 +56,7 @@ def xmliter(obj, nodename):
 def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
     from lxml import etree
     reader = _StreamReader(obj)
-    tag = f'{{{namespace}}}{nodename}'if namespace else nodename
+    tag = f'{{{namespace}}}{nodename}' if namespace else nodename
     iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
     selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename)
     for _, node in iterable:

From 6050604f626a5ee38239ef1eff44d0c867469a3b Mon Sep 17 00:00:00 2001
From: GeorgeA92 <george.zatserklyany@gmail.com>
Date: Tue, 6 Oct 2020 18:59:57 +0300
Subject: [PATCH 166/716] httocompression/response_bytes tests added

---
 tests/test_downloadermiddleware_httpcompression.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 998749c2e..144fd3378 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -77,6 +77,7 @@ class HttpCompressionTest(TestCase):
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
         self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
 
     def test_process_response_br(self):
         try:
@@ -91,6 +92,7 @@ class HttpCompressionTest(TestCase):
         assert newresponse.body.startswith(b"<!DOCTYPE")
         assert 'Content-Encoding' not in newresponse.headers
         self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
 
     def test_process_response_rawdeflate(self):
         response = self._getresponse('rawdeflate')
@@ -102,6 +104,7 @@ class HttpCompressionTest(TestCase):
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
         self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74840)
 
     def test_process_response_zlibdelate(self):
         response = self._getresponse('zlibdeflate')
@@ -113,6 +116,7 @@ class HttpCompressionTest(TestCase):
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
         self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74840)
 
     def test_process_response_plain(self):
         response = Response('http://scrapytest.org', body=b'<!DOCTYPE...')
@@ -123,6 +127,7 @@ class HttpCompressionTest(TestCase):
         assert newresponse is response
         assert newresponse.body.startswith(b'<!DOCTYPE')
         self.assertStatsEqual('httpcompression/response_count', None)
+        self.assertStatsEqual('httpcompression/response_bytes', None)
 
     def test_multipleencodings(self):
         response = self._getresponse('gzip')
@@ -151,6 +156,7 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
         self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 104)
 
     def test_process_response_force_recalculate_encoding(self):
         headers = {
@@ -171,6 +177,7 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
         self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 104)
 
     def test_process_response_no_content_type_header(self):
         headers = {
@@ -187,6 +194,7 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(newresponse.body, plainbody)
         self.assertEqual(newresponse.encoding, resolve_encoding('gb2312'))
         self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 104)
 
     def test_process_response_gzipped_contenttype(self):
         response = self._getresponse('gzip')
@@ -198,6 +206,7 @@ class HttpCompressionTest(TestCase):
         self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
         self.assertNotIn('Content-Encoding', newresponse.headers)
         self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
 
     def test_process_response_gzip_app_octetstream_contenttype(self):
         response = self._getresponse('gzip')
@@ -209,6 +218,7 @@ class HttpCompressionTest(TestCase):
         self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
         self.assertNotIn('Content-Encoding', newresponse.headers)
         self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
 
     def test_process_response_gzip_binary_octetstream_contenttype(self):
         response = self._getresponse('x-gzip')
@@ -220,6 +230,7 @@ class HttpCompressionTest(TestCase):
         self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
         self.assertNotIn('Content-Encoding', newresponse.headers)
         self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
 
     def test_process_response_gzipped_gzip_file(self):
         """Test that a gzip Content-Encoded .gz file is gunzipped
@@ -263,6 +274,7 @@ class HttpCompressionTest(TestCase):
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertEqual(gunzip(newresponse.body), plainbody)
         self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 230)
 
     def test_process_response_head_request_no_decode_required(self):
         response = self._getresponse('gzip')
@@ -274,3 +286,4 @@ class HttpCompressionTest(TestCase):
         self.assertIs(newresponse, response)
         self.assertEqual(response.body, b'')
         self.assertStatsEqual('httpcompression/response_count', None)
+        self.assertStatsEqual('httpcompression/response_bytes', None)

From e40788153ca7dee9e0d4f8ac16a9e17278f79058 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 6 Oct 2020 19:13:29 +0200
Subject: [PATCH 167/716] Do not consider about: URLs invalid

---
 scrapy/http/request/__init__.py | 6 +++++-
 tests/test_http_request.py      | 9 +++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index ef58deacc..498f1b052 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -65,7 +65,11 @@ class Request(object_ref):
         s = safe_url_string(url, self.encoding)
         self._url = escape_ajax(s)
 
-        if ('://' not in self._url) and (not self._url.startswith('data:')):
+        if (
+            '://' not in self._url
+            and not self._url.startswith('about:')
+            and not self._url.startswith('data:')
+        ):
             raise ValueError(f'Missing scheme in request url: {self._url}')
 
     url = property(_get_url, obsolete_setter(_set_url, 'url'))
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 0a303dbe2..74579dfc4 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -43,6 +43,15 @@ class RequestTest(unittest.TestCase):
         assert r.headers is not headers
         self.assertEqual(r.headers[b"caca"], b"coco")
 
+    def test_url_scheme(self):
+        # This test passes by not raising any (ValueError) exception
+        self.request_class('http://example.org')
+        self.request_class('https://example.org')
+        self.request_class('s3://example.org')
+        self.request_class('ftp://example.org')
+        self.request_class('about:config')
+        self.request_class('data:,Hello%2C%20World!')
+
     def test_url_no_scheme(self):
         self.assertRaises(ValueError, self.request_class, 'foo')
         self.assertRaises(ValueError, self.request_class, '/foo/')

From 9f02df20c51c80739d2362c388930ad93e3de34a Mon Sep 17 00:00:00 2001
From: dswij <dharmasw@outlook.com>
Date: Wed, 7 Oct 2020 01:10:01 +0700
Subject: [PATCH 168/716] Remove PyDispatcher from general requirements

---
 setup.py |  2 +-
 tox.ini  | 15 +++++++++++----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/setup.py b/setup.py
index 0c2281400..c046684c0 100644
--- a/setup.py
+++ b/setup.py
@@ -24,7 +24,6 @@ install_requires = [
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
     'parsel>=1.5.0',
-    'PyDispatcher>=2.0.5',
     'pyOpenSSL>=16.2.0',
     'queuelib>=1.4.2',
     'service_identity>=16.0.0',
@@ -38,6 +37,7 @@ extras_require = {}
 if has_environment_marker_platform_impl_support():
     extras_require[':platform_python_implementation == "CPython"'] = [
         'lxml>=3.5.0',
+        'PyDispatcher>=2.0.5',
     ]
     extras_require[':platform_python_implementation == "PyPy"'] = [
         # Earlier lxml versions are affected by
diff --git a/tox.ini b/tox.ini
index 4e6ca3ec9..d29cea0fc 100644
--- a/tox.ini
+++ b/tox.ini
@@ -72,7 +72,6 @@ deps =
     itemadapter==0.1.0
     parsel==1.5.0
     Protego==0.1.15
-    PyDispatcher==2.0.5
     #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>=
     pyOpenSSL==18.0.0
     queuelib==1.4.2
@@ -89,7 +88,12 @@ deps =
 [testenv:pinned]
 deps =
     {[pinned]deps}
-    lxml==3.5.0
+    PyDispatcher==2.0.5
+
+[testenv: pypy-pinned]
+deps = 
+    {[pinned]deps}
+    PyPyDispatcher==2.1.0
 
 [testenv:windows-pinned]
 basepython = python3
@@ -111,13 +115,16 @@ commands =
 
 [testenv:asyncio-pinned]
 commands = {[testenv:asyncio]commands}
-deps = {[testenv:pinned]deps}
+deps = 
+    {[testenv:pinned]deps}
+    lxml==3.5.0
 
 [testenv:pypy3]
 basepython = pypy3
 deps = 
     {[testenv]deps}
-    PyDispatcher==2.0.5
+    PyPyDispatcher>=2.1.0
+    lxml==4.0.0
 commands =
     py.test {posargs:--durations=10 docs scrapy tests}
 

From 2e734e6b35f686f62dea11e9484c7646c7c49ca8 Mon Sep 17 00:00:00 2001
From: "P. Chen" <pengyu@libstarrify.so>
Date: Tue, 6 Oct 2020 19:51:05 +0100
Subject: [PATCH 169/716] Minor update on the import order in
 scrapy/downloadermiddlewares/httpcompression.py

---
 scrapy/downloadermiddlewares/httpcompression.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 56421a6ba..f504302e2 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,5 +1,5 @@
-import zlib
 import io
+import zlib
 
 from scrapy.utils.gz import gunzip
 from scrapy.http import Response, TextResponse

From 156bb0a1d413c7c6acafc30003ef98030a06e47f Mon Sep 17 00:00:00 2001
From: "P. Chen" <pengyu@libstarrify.so>
Date: Tue, 6 Oct 2020 19:53:40 +0100
Subject: [PATCH 170/716] Fixing the minor typo on test file path in
 tests/test_downloadermiddleware_httpcompression.py

---
 tests/test_downloadermiddleware_httpcompression.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 7e9f9cc5d..4c5bfc577 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -25,7 +25,7 @@ FORMAT = {
     # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin
     'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'),
     # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin
-    'zstd-streaming-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'),
+    'zstd-streaming-no-content-size': ('html-zstd-streaming-no-content-size.bin', 'zstd'),
 }
 
 

From 1a597d5e3dda7a467d69cff51df7e731f2f2d6b5 Mon Sep 17 00:00:00 2001
From: OfirD1 <beyblade1234@walla.com>
Date: Tue, 6 Oct 2020 21:54:42 +0300
Subject: [PATCH 171/716] moved the sentence about processing pending requests
 when a spider is closed onto a generic note.

---
 docs/topics/extensions.rst | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 14096ada4..519f18b63 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -257,6 +257,12 @@ settings:
 * :setting:`CLOSESPIDER_PAGECOUNT`
 * :setting:`CLOSESPIDER_ERRORCOUNT`
 
+.. note::
+
+   When a certain closing condition is met, requests which are 
+   currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` 
+   requests) are still processed.
+
 .. setting:: CLOSESPIDER_TIMEOUT
 
 CLOSESPIDER_TIMEOUT
@@ -279,8 +285,6 @@ Default: ``0``
 An integer which specifies a number of items. If the spider scrapes more than
 that amount and those items are passed by the item pipeline, the
 spider will be closed with the reason ``closespider_itemcount``.
-Requests which  are currently in the downloader queue (up to
-:setting:`CONCURRENT_REQUESTS` requests) are still processed.
 If zero (or non set), spiders won't be closed by number of passed items.
 
 .. setting:: CLOSESPIDER_PAGECOUNT

From 9461414b14a7444dd52eda517dc5e373f79c2b7e Mon Sep 17 00:00:00 2001
From: dswij <dharmasw@outlook.com>
Date: Wed, 7 Oct 2020 11:26:53 +0700
Subject: [PATCH 172/716] minor changes to remove unnecessary lines

---
 .travis.yml |  1 -
 tox.ini     | 19 +++++++------------
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index fb4802070..b883c5b78 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -48,7 +48,6 @@ install:
         tar -jxf ${PYPY_VERSION}.tar.bz2
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-        $HOME/virtualenvs/$PYPY_VERSION/bin/pypy -m ensurepip --default-pip
       fi
   - pip install -U tox twine wheel codecov
 
diff --git a/tox.ini b/tox.ini
index d29cea0fc..5dc99621e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -72,12 +72,14 @@ deps =
     itemadapter==0.1.0
     parsel==1.5.0
     Protego==0.1.15
+    !pypy3: PyDispatcher==2.0.5
     #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>=
     pyOpenSSL==18.0.0
     queuelib==1.4.2
     service_identity==16.0.0
     Twisted==20.3.0
     w3lib==1.17.0
+    #zope.interface==5.1.2 to resolve conflict with Twisted==20.3.0
     zope.interface==5.1.2
     -rtests/requirements-py3.txt
     # Extras
@@ -88,13 +90,8 @@ deps =
 [testenv:pinned]
 deps =
     {[pinned]deps}
-    PyDispatcher==2.0.5
-
-[testenv: pypy-pinned]
-deps = 
-    {[pinned]deps}
-    PyPyDispatcher==2.1.0
-
+    lxml==3.5.0
+    
 [testenv:windows-pinned]
 basepython = python3
 deps =
@@ -114,26 +111,24 @@ commands =
     {[testenv]commands} --reactor=asyncio
 
 [testenv:asyncio-pinned]
+deps = {[testenv:pinned]deps}
 commands = {[testenv:asyncio]commands}
-deps = 
-    {[testenv:pinned]deps}
-    lxml==3.5.0
 
 [testenv:pypy3]
 basepython = pypy3
 deps = 
     {[testenv]deps}
-    PyPyDispatcher>=2.1.0
     lxml==4.0.0
 commands =
     py.test {posargs:--durations=10 docs scrapy tests}
 
 [testenv:pypy3-pinned]
 basepython = {[testenv:pypy3]basepython}
-commands = {[testenv:pypy3]commands}
 deps =
     {[pinned]deps}
     lxml==4.0.0
+    PyPyDispatcher==2.1.0
+commands = {[testenv:pypy3]commands}
 
 [docs]
 changedir = docs

From 13ae17aecc632dc03af1bd8b2b63e48301147cc2 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 8 Oct 2020 14:04:52 -0300
Subject: [PATCH 173/716] Add xfail_strict=true to pytest.ini

---
 pytest.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytest.ini b/pytest.ini
index ca8191f42..1c95f715a 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,4 +1,5 @@
 [pytest]
+xfail_strict = true
 usefixtures = chdir
 python_files=test_*.py __init__.py
 python_classes=
@@ -40,4 +41,3 @@ flake8-ignore =
     scrapy/utils/multipart.py F403
     scrapy/utils/url.py F403 F405
     tests/test_loader.py E741
-    

From b55c911ddc21a41a6e4204aaf239a16e892de7b5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 21 Sep 2020 10:59:55 -0300
Subject: [PATCH 174/716] Fix CachingHostnameResolver

---
 scrapy/resolver.py | 57 +++++++++++++++++++++++++++-------------------
 1 file changed, 34 insertions(+), 23 deletions(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index f191deac6..0350c82b9 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -1,4 +1,5 @@
 from twisted.internet import defer
+from twisted.internet._resolver import HostResolution
 from twisted.internet.base import ThreadedResolver
 from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple
 from zope.interface.declarations import implementer, provider
@@ -50,6 +51,27 @@ class CachingThreadedResolver(ThreadedResolver):
         return result
 
 
+@provider(IResolutionReceiver)
+class _CachingResolutionReceiver:
+    def __init__(self, resolutionReceiver, hostName):
+        self.resolutionReceiver = resolutionReceiver
+        self.hostName = hostName
+        self.addresses = []
+
+    def resolutionBegan(self, resolution):
+        self.resolutionReceiver.resolutionBegan(resolution)
+        self.resolution = resolution
+
+    def addressResolved(self, address):
+        self.resolutionReceiver.addressResolved(address)
+        self.addresses.append(address)
+
+    def resolutionComplete(self):
+        self.resolutionReceiver.resolutionComplete()
+        if self.addresses:
+            dnscache[self.hostName] = self.addresses
+
+
 @implementer(IHostnameResolver)
 class CachingHostnameResolver:
     """
@@ -73,33 +95,22 @@ class CachingHostnameResolver:
     def install_on_reactor(self):
         self.reactor.installNameResolver(self)
 
-    def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
-                        addressTypes=None, transportSemantics='TCP'):
-
-        @provider(IResolutionReceiver)
-        class CachingResolutionReceiver(resolutionReceiver):
-
-            def resolutionBegan(self, resolution):
-                super().resolutionBegan(resolution)
-                self.resolution = resolution
-                self.resolved = False
-
-            def addressResolved(self, address):
-                super().addressResolved(address)
-                self.resolved = True
-
-            def resolutionComplete(self):
-                super().resolutionComplete()
-                if self.resolved:
-                    dnscache[hostName] = self.resolution
-
+    def resolveHostName(
+        self, resolutionReceiver, hostName, portNumber=0, addressTypes=None, transportSemantics="TCP"
+    ):
         try:
-            return dnscache[hostName]
+            addresses = dnscache[hostName]
         except KeyError:
             return self.original_resolver.resolveHostName(
-                CachingResolutionReceiver(),
+                _CachingResolutionReceiver(resolutionReceiver, hostName),
                 hostName,
                 portNumber,
                 addressTypes,
-                transportSemantics
+                transportSemantics,
             )
+        else:
+            resolutionReceiver.resolutionBegan(HostResolution(hostName))
+            for addr in addresses:
+                resolutionReceiver.addressResolved(addr)
+            resolutionReceiver.resolutionComplete()
+            return resolutionReceiver

From 8fe5876597825a4df03bd15e6b1cd1682b806de8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 30 Sep 2020 14:56:17 -0300
Subject: [PATCH 175/716] HostResolution implementation

---
 scrapy/resolver.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 0350c82b9..0bef555a6 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -1,7 +1,6 @@
 from twisted.internet import defer
-from twisted.internet._resolver import HostResolution
 from twisted.internet.base import ThreadedResolver
-from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple
+from twisted.internet.interfaces import IHostResolution, IHostnameResolver, IResolutionReceiver, IResolverSimple
 from zope.interface.declarations import implementer, provider
 
 from scrapy.utils.datatypes import LocalCache
@@ -51,6 +50,15 @@ class CachingThreadedResolver(ThreadedResolver):
         return result
 
 
+@implementer(IHostResolution)
+class HostResolution:
+    def __init__(self, name):
+        self.name = name
+
+    def cancel(self):
+        raise NotImplementedError()
+
+
 @provider(IResolutionReceiver)
 class _CachingResolutionReceiver:
     def __init__(self, resolutionReceiver, hostName):

From 868826b346714ceff821886536a3b259072f8396 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 2 Oct 2020 15:16:58 -0300
Subject: [PATCH 176/716] CachingHostnameResolver tests

---
 .../alternative_name_resolver.py              | 15 ----------
 .../caching_hostname_resolver.py              | 30 +++++++++++++++++++
 .../caching_hostname_resolver_ipv6.py         | 19 ++++++++++++
 tests/CrawlerProcess/default_name_resolver.py | 11 +++++--
 tests/test_crawler.py                         | 20 +++++++++----
 5 files changed, 72 insertions(+), 23 deletions(-)
 delete mode 100644 tests/CrawlerProcess/alternative_name_resolver.py
 create mode 100644 tests/CrawlerProcess/caching_hostname_resolver.py
 create mode 100644 tests/CrawlerProcess/caching_hostname_resolver_ipv6.py

diff --git a/tests/CrawlerProcess/alternative_name_resolver.py b/tests/CrawlerProcess/alternative_name_resolver.py
deleted file mode 100644
index 2c466da04..000000000
--- a/tests/CrawlerProcess/alternative_name_resolver.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import scrapy
-from scrapy.crawler import CrawlerProcess
-
-
-class IPv6Spider(scrapy.Spider):
-    name = "ipv6_spider"
-    start_urls = ["http://[::1]"]
-
-
-process = CrawlerProcess(settings={
-    "RETRY_ENABLED": False,
-    "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
-})
-process.crawl(IPv6Spider)
-process.start()
diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py
new file mode 100644
index 000000000..f9eab3543
--- /dev/null
+++ b/tests/CrawlerProcess/caching_hostname_resolver.py
@@ -0,0 +1,30 @@
+import sys
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class CachingHostnameResolverSpider(scrapy.Spider):
+    """
+    Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution)
+    """
+    name = "caching_hostname_resolver_spider"
+
+    def start_requests(self):
+        yield scrapy.Request(self.url)
+
+    def parse(self, response):
+        for _ in range(10):
+            yield scrapy.Request(response.url, dont_filter=True, callback=self.ignore_response)
+
+    def ignore_response(self, response):
+        self.logger.info(repr(response.ip_address))
+
+
+if __name__ == "__main__":
+    process = CrawlerProcess(settings={
+        "RETRY_ENABLED": False,
+        "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+    })
+    process.crawl(CachingHostnameResolverSpider, url=sys.argv[1])
+    process.start()
diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
new file mode 100644
index 000000000..3340d2f84
--- /dev/null
+++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
@@ -0,0 +1,19 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class CachingHostnameResolverSpider(scrapy.Spider):
+    """
+    Finishes without a twisted.internet.error.DNSLookupError exception
+    """
+    name = "caching_hostname_resolver_spider"
+    start_urls = ["http://[::1]"]
+
+
+if __name__ == "__main__":
+    process = CrawlerProcess(settings={
+        "RETRY_ENABLED": False,
+        "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+    })
+    process.crawl(CachingHostnameResolverSpider)
+    process.start()
diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py
index 60d91b68b..05a98fbec 100644
--- a/tests/CrawlerProcess/default_name_resolver.py
+++ b/tests/CrawlerProcess/default_name_resolver.py
@@ -3,10 +3,15 @@ from scrapy.crawler import CrawlerProcess
 
 
 class IPv6Spider(scrapy.Spider):
+    """
+    Raises a twisted.internet.error.DNSLookupError:
+    the default name resolver does not handle IPv6 addresses.
+    """
     name = "ipv6_spider"
     start_urls = ["http://[::1]"]
 
 
-process = CrawlerProcess(settings={"RETRY_ENABLED": False})
-process.crawl(IPv6Spider)
-process.start()
+if __name__ == "__main__":
+    process = CrawlerProcess(settings={"RETRY_ENABLED": False})
+    process.crawl(IPv6Spider)
+    process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 85035a220..246e54860 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -22,6 +22,8 @@ from scrapy.extensions.throttle import AutoThrottle
 from scrapy.extensions import telnet
 from scrapy.utils.test import get_testenv
 
+from tests.mockserver import MockServer
+
 
 class BaseCrawlerTest(unittest.TestCase):
 
@@ -280,9 +282,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
 
 class ScriptRunnerMixin:
-    def run_script(self, script_name):
+    def run_script(self, script_name, *script_args):
         script_path = os.path.join(self.script_dir, script_name)
-        args = (sys.executable, script_path)
+        args = [sys.executable, script_path] + list(script_args)
         p = subprocess.Popen(args, env=get_testenv(),
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         stdout, stderr = p.communicate()
@@ -321,11 +323,19 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
             log)
 
-    def test_ipv6_alternative_name_resolver(self):
-        log = self.run_script('alternative_name_resolver.py')
-        self.assertIn('Spider closed (finished)', log)
+    def test_caching_hostname_resolver_ipv6(self):
+        log = self.run_script("caching_hostname_resolver_ipv6.py")
+        self.assertIn("Spider closed (finished)", log)
         self.assertNotIn("twisted.internet.error.DNSLookupError", log)
 
+    def test_caching_hostname_resolver_finite_execution(self):
+        with MockServer() as mock_server:
+            log = self.run_script("caching_hostname_resolver.py", mock_server.http_address)
+            self.assertIn("Spider closed (finished)", log)
+            self.assertNotIn("ERROR: Error downloading", log)
+            self.assertNotIn("TimeoutError", log)
+            self.assertNotIn("twisted.internet.error.DNSLookupError", log)
+
     def test_reactor_select(self):
         log = self.run_script("twisted_reactor_select.py")
         self.assertIn("Spider closed (finished)", log)

From 015c82b974841897a637dcd3ddbd40ab48a70ecf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sun, 11 Oct 2020 22:09:45 +0200
Subject: [PATCH 177/716] Scrapy 2.4 release notes (#4808)

---
 docs/news.rst                  | 306 ++++++++++++++++++++++++++++++++-
 docs/topics/asyncio.rst        |   2 +
 docs/topics/feed-exports.rst   |  18 +-
 docs/topics/media-pipeline.rst |  31 ++--
 docs/topics/settings.rst       |   2 +-
 5 files changed, 344 insertions(+), 15 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 850b323ef..d5c342c86 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,310 @@
 Release notes
 =============
 
+.. _release-2.4.0:
+
+Scrapy 2.4.0 (2020-10-??)
+-------------------------
+
+Highlights:
+
+*   Python 3.5 support has been dropped.
+
+*   The ``file_path`` method of :ref:`media pipelines <topics-media-pipeline>`
+    can now access the source :ref:`item <topics-items>`.
+
+    This allows you to set a download file path based on item data.
+
+*   The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows
+    to define keyword parameters to pass to :ref:`item exporter classes
+    <topics-exporters>`
+
+*   You can now choose whether :ref:`feed exports <topics-feed-exports>`
+    overwrite or append to the output file.
+
+    For example, when using the :command:`crawl` or :command:`runspider`
+    commands, you can use the ``-O`` option instead of ``-o`` to overwrite the
+    output file.
+
+*   Zstd-compressed responses are now supported if zstandard_ is installed.
+
+*   In settings, where the import path of a class is required, it is now
+    possible to pass a class object instead.
+
+Modified requirements
+~~~~~~~~~~~~~~~~~~~~~
+
+*   Python 3.6 or greater is now required; support for Python 3.5 has been
+    dropped
+
+    As a result:
+
+    -   When using PyPy, PyPy 7.2.0 or greater :ref:`is now required
+        <faq-python-versions>`
+
+    -   For Amazon S3 storage support in :ref:`feed exports
+        <topics-feed-storage-s3>` or :ref:`media pipelines
+        <media-pipelines-s3>`, botocore_ 1.4.87 or greater is now required
+
+    -   To use the :ref:`images pipeline <images-pipeline>`, Pillow_ 4.0.0 or
+        greater is now required
+
+    (:issue:`4718`, :issue:`4732`, :issue:`4733`, :issue:`4742`, :issue:`4743`,
+    :issue:`4764`)
+
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+*   :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again
+    discards cookies defined in :attr:`Request.headers
+    <scrapy.http.Request.headers>`.
+
+    We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it
+    was reported that the current implementation could break existing code.
+
+    If you need to set cookies for a request, use the :class:`Request.cookies
+    <scrapy.http.Request>` parameter.
+
+    A future version of Scrapy will include a new, better implementation of the
+    reverted bug fix.
+
+    (:issue:`4717`, :issue:`4823`)
+
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+*   :class:`scrapy.extensions.feedexport.S3FeedStorage` no longer reads the
+    values of ``access_key`` and ``secret_key`` from the running project
+    settings when they are not passed to its ``__init__`` method; you must
+    either pass those parameters to its ``__init__`` method or use
+    :class:`S3FeedStorage.from_crawler
+    <scrapy.extensions.feedexport.S3FeedStorage.from_crawler>`
+    (:issue:`4356`, :issue:`4411`, :issue:`4688`)
+
+*   :attr:`Rule.process_request <scrapy.spiders.crawl.Rule.process_request>`
+    no longer admits callables which expect a single ``request`` parameter,
+    rather than both ``request`` and ``response`` (:issue:`4818`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+*   In custom :ref:`media pipelines <topics-media-pipeline>`, signatures that
+    do not accept a keyword-only ``item`` parameter in any of the  methods that
+    :ref:`now support this parameter <media-pipeline-item-parameter>` are now
+    deprecated (:issue:`4628`, :issue:`4686`)
+
+*   In custom :ref:`feed storage backend classes <topics-feed-storage>`,
+    ``__init__`` method signatures that do not accept a keyword-only
+    ``feed_options`` parameter are now deprecated (:issue:`547`, :issue:`716`,
+    :issue:`4512`)
+
+*   The :class:`scrapy.utils.python.WeakKeyCache` class is now deprecated
+    (:issue:`4684`, :issue:`4701`)
+
+*   The :func:`scrapy.utils.boto.is_botocore` function is now deprecated, use
+    :func:`scrapy.utils.boto.is_botocore_available` instead (:issue:`4734`,
+    :issue:`4776`)
+
+
+New features
+~~~~~~~~~~~~
+
+.. _media-pipeline-item-parameter:
+
+*   The following methods of :ref:`media pipelines <topics-media-pipeline>` now
+    accept an ``item`` keyword-only parameter containing the source
+    :ref:`item <topics-items>`:
+
+    -   In :class:`scrapy.pipelines.files.FilesPipeline`:
+
+        -   :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded`
+
+        -   :meth:`~scrapy.pipelines.files.FilesPipeline.file_path`
+
+        -   :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded`
+
+        -   :meth:`~scrapy.pipelines.files.FilesPipeline.media_to_download`
+
+    -   In :class:`scrapy.pipelines.images.ImagesPipeline`:
+
+        -   :meth:`~scrapy.pipelines.images.ImagesPipeline.file_downloaded`
+
+        -   :meth:`~scrapy.pipelines.images.ImagesPipeline.file_path`
+
+        -   :meth:`~scrapy.pipelines.images.ImagesPipeline.get_images`
+
+        -   :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded`
+
+        -   :meth:`~scrapy.pipelines.images.ImagesPipeline.media_downloaded`
+
+        -   :meth:`~scrapy.pipelines.images.ImagesPipeline.media_to_download`
+
+    (:issue:`4628`, :issue:`4686`)
+
+*   The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows
+    to define keyword parameters to pass to :ref:`item exporter classes
+    <topics-exporters>` (:issue:`4606`, :issue:`4768`)
+
+*   :ref:`Feed exports <topics-feed-exports>` gained overwrite support:
+
+    *   When using the :command:`crawl` or :command:`runspider` commands, you
+        can use the ``-O`` option instead of ``-o`` to overwrite the output
+        file
+
+    *   You can use the ``overwrite`` key in the :setting:`FEEDS` setting to
+        configure whether to overwrite the output file (``True``) or append to
+        its content (``False``)
+
+    *   The ``__init__`` and ``from_crawler`` methods of :ref:`feed storage
+        backend classes <topics-feed-storage>` now receive a new keyword-only
+        parameter, ``feed_options``, which is a dictionary of :ref:`feed
+        options <feed-options>`
+
+    (:issue:`547`, :issue:`716`, :issue:`4512`)
+
+*   Zstd-compressed responses are now supported if zstandard_ is installed
+    (:issue:`4831`)
+
+*   In settings, where the import path of a class is required, it is now
+    possible to pass a class object instead (:issue:`3870`, :issue:`3873`).
+
+    This includes also settings where only part of its value is made of an
+    import path, such as :setting:`DOWNLOADER_MIDDLEWARES` or
+    :setting:`DOWNLOAD_HANDLERS`.
+
+*   :ref:`Downloader middlewares <topics-downloader-middleware>` can now
+    override :class:`response.request <scrapy.http.Response.request>`.
+
+    If a :ref:`downloader middleware <topics-downloader-middleware>` returns
+    a :class:`~scrapy.http.Response` object from
+    :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`
+    or
+    :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`
+    with a custom :class:`~scrapy.http.Request` object assigned to
+    :class:`response.request <scrapy.http.Response.request>`:
+
+    -   The response is handled by the callback of that custom
+        :class:`~scrapy.http.Request` object, instead of being handled by the
+        callback of the original :class:`~scrapy.http.Request` object
+
+    -   That custom :class:`~scrapy.http.Request` object is now sent as the
+        ``request`` argument to the :signal:`response_received` signal, instead
+        of the original :class:`~scrapy.http.Request` object
+
+    (:issue:`4529`, :issue:`4632`)
+
+*   When using the :ref:`FTP feed storage backend <topics-feed-storage-ftp>`:
+
+    -   It is now possible to set the new ``overwrite`` :ref:`feed option
+        <feed-options>` to ``False`` to append to an existing file instead of
+        overwriting it
+
+    -   The FTP password can now be omitted if it is not necessary
+
+    (:issue:`547`, :issue:`716`, :issue:`4512`)
+
+*   The ``__init__`` method of :class:`~scrapy.exporters.CsvItemExporter` now
+    supports an ``errors`` parameter to indicate how to handle encoding errors
+    (:issue:`4755`)
+
+*   When :ref:`using asyncio <using-asyncio>`, it is now possible to
+    :ref:`set a custom asyncio loop <using-custom-loops>` (:issue:`4306`,
+    :issue:`4414`)
+
+*   Serialized requests (see :ref:`topics-jobs`) now support callbacks that are
+    spider methods that delegate on other callable (:issue:`4756`)
+
+*   When a response is larger than :setting:`DOWNLOAD_MAXSIZE`, the logged
+    message is now a warning, instead of an error (:issue:`3874`,
+    :issue:`3886`, :issue:`4752`)
+
+
+Bug fixes
+~~~~~~~~~
+
+*   The :command:`genspider` command no longer overwrites existing files
+    unless the ``--force`` option is used (:issue:`4561`, :issue:`4616`,
+    :issue:`4623`)
+
+*   Cookies with an empty value are no longer considered invalid cookies
+    (:issue:`4772`)
+
+*   The :command:`runspider` command now supports files with the ``.pyw`` file
+    extension (:issue:`4643`, :issue:`4646`)
+
+*   The :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
+    middleware now simply ignores unsupported proxy values (:issue:`3331`,
+    :issue:`4778`)
+
+*   Checks for generator callbacks with a ``return`` statement no longer warn
+    about ``return`` statements in nested functions (:issue:`4720`,
+    :issue:`4721`)
+
+*   The system file mode creation mask no longer affects the permissions of
+    files generated using the :command:`startproject` command (:issue:`4722`)
+
+*   :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names
+    (:issue:`861`, :issue:`4746`)
+
+*   :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can
+    work when using a headless browser (:issue:`4835`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+*   The :setting:`FEED_URI_PARAMS` setting is now documented (:issue:`4671`,
+    :issue:`4724`)
+
+*   Improved the documentation of
+    :ref:`link extractors <topics-link-extractors>` with an usage example from
+    a spider callback and reference documentation for the
+    :class:`~scrapy.link.Link` class (:issue:`4751`, :issue:`4775`)
+
+*   Clarified the impact of :setting:`CONCURRENT_REQUESTS` when using the
+    :class:`~scrapy.extensions.closespider.CloseSpider` extension
+    (:issue:`4836`)
+
+*   Removed references to Python 2’s ``unicode`` type (:issue:`4547`,
+    :issue:`4703`)
+
+*   We now have an :ref:`official deprecation policy <deprecation-policy>`
+    (:issue:`4705`)
+
+*   Our :ref:`documentation policies <documentation-policies>` now cover usage
+    of Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged`
+    directives, and we have removed usages referencing Scrapy 1.4.0 and earlier
+    versions (:issue:`3971`, :issue:`4310`)
+
+*   Other documentation cleanups (:issue:`4090`, :issue:`4782`, :issue:`4800`,
+    :issue:`4801`, :issue:`4809`, :issue:`4816`, :issue:`4825`)
+
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+*   Extended typing hints (:issue:`4243`, :issue:`4691`)
+
+*   Added tests for the :command:`check` command (:issue:`4663`)
+
+*   Fixed test failures on Debian (:issue:`4726`, :issue:`4727`, :issue:`4735`)
+
+*   Improved Windows test coverage (:issue:`4723`)
+
+*   Switched to :ref:`formatted string literals <f-strings>` where possible
+    (:issue:`4307`, :issue:`4324`, :issue:`4672`)
+
+*   Modernized :func:`super` usage (:issue:`4707`)
+
+*   Other code and test cleanups (:issue:`1790`, :issue:`3288`, :issue:`4165`,
+    :issue:`4564`, :issue:`4651`, :issue:`4714`, :issue:`4738`, :issue:`4745`,
+    :issue:`4747`, :issue:`4761`, :issue:`4765`, :issue:`4804`, :issue:`4817`,
+    :issue:`4820`, :issue:`4822`, :issue:`4839`)
+
+
 .. _release-2.3.0:
 
 Scrapy 2.3.0 (2020-08-04)
@@ -4008,9 +4312,9 @@ First release of Scrapy.
 .. _six: https://six.readthedocs.io/
 .. _tox: https://pypi.org/project/tox/
 .. _Twisted: https://twistedmatrix.com/trac/
-.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
 .. _w3lib: https://github.com/scrapy/w3lib
 .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
 .. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
 .. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/
 .. _Zsh: https://www.zsh.org/
+.. _zstandard: https://pypi.org/project/zstandard/
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index bfb430d52..91e1cca0d 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -1,3 +1,5 @@
+.. _using-asyncio:
+
 =======
 asyncio
 =======
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 1744cfd74..843ed25f9 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -184,7 +184,7 @@ The feeds are stored on `Amazon S3`_.
    * ``s3://mybucket/path/to/export.csv``
    * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
 
- * Required external libraries: `botocore`_
+ * Required external libraries: `botocore`_ >= 1.4.87
 
 The AWS credentials can be passed as user/password in the URI, or they can be
 passed through the following settings:
@@ -319,6 +319,8 @@ For instance::
         },
     }
 
+.. _feed-options:
+
 The following is a list of the accepted keys and the setting that is used
 as a fallback value if that key is not provided for a specific feed definition:
 
@@ -329,6 +331,8 @@ as a fallback value if that key is not provided for a specific feed definition:
 -   ``batch_item_count``: falls back to
     :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
 
+    .. versionadded:: 2.3.0
+
 -   ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.
 
 -   ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
@@ -337,6 +341,8 @@ as a fallback value if that key is not provided for a specific feed definition:
 
 -   ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
 
+    .. versionadded:: 2.4.0
+
 -   ``overwrite``: whether to overwrite the file if it already exists
     (``True``) or append to its content (``False``).
 
@@ -355,6 +361,8 @@ as a fallback value if that key is not provided for a specific feed definition:
 
     -   :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
 
+    .. versionadded:: 2.4.0
+
 -   ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
 
 -   ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
@@ -517,7 +525,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
 .. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
 
 FEED_EXPORT_BATCH_ITEM_COUNT
------------------------------
+----------------------------
+
+.. versionadded:: 2.3.0
 
 Default: ``0``
 
@@ -586,11 +596,15 @@ The function signature should be as follows:
             If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id``
             is always ``1``.
 
+            .. versionadded:: 2.3.0
+
         -   ``batch_time``: UTC date and time, in ISO format with ``:``
             replaced with ``-``.
 
             See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
 
+            .. versionadded:: 2.3.0
+
         -   ``time``: ``batch_time``, with microseconds set to ``0``.
    :type params: dict
 
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 06809c24b..156897274 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -56,6 +56,8 @@ this:
    error will be logged and the file won't be present in the ``files`` field.
 
 
+.. _images-pipeline:
+
 Using the Images Pipeline
 =========================
 
@@ -68,14 +70,10 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
 can configure some extra functions like generating thumbnails and filtering
 the images based on their size.
 
-The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to
-JPEG/RGB format, so you need to install this library in order to use it.
-`Python Imaging Library`_ (PIL) should also work in most cases, but it is known
-to cause troubles in some setups, so we recommend to use `Pillow`_ instead of
-PIL.
+The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
+thumbnailing and normalizing images to JPEG/RGB format.
 
 .. _Pillow: https://github.com/python-pillow/Pillow
-.. _Python Imaging Library: http://www.pythonware.com/products/pil/
 
 
 .. _topics-media-pipeline-enabling:
@@ -164,14 +162,17 @@ FTP supports two different connection modes: active or passive. Scrapy uses
 the passive connection mode by default. To use the active connection mode instead,
 set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
 
+.. _media-pipelines-s3:
+
 Amazon S3 storage
 -----------------
 
 .. setting:: FILES_STORE_S3_ACL
 .. setting:: IMAGES_STORE_S3_ACL
 
-:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3
-bucket. Scrapy will automatically upload the files to the bucket.
+If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
+:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
+automatically upload the files to the bucket.
 
 For example, this is a valid :setting:`IMAGES_STORE` value::
 
@@ -187,8 +188,9 @@ policy::
 
 For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
 
-Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
-self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
+You can also use other S3-like storages. Storages like self-hosted `Minio`_ or
+`s3.scality`_. All you need to do is set endpoint option in you Scrapy
+settings::
 
     AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
 
@@ -197,9 +199,10 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
     AWS_USE_SSL = False # or True (None by default)
     AWS_VERIFY = False # or True (None by default)
 
+.. _botocore: https://github.com/boto/botocore
+.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 .. _Minio: https://github.com/minio/minio
 .. _s3.scality: https://s3.scality.com/
-.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 
 .. _media-pipeline-gcs:
@@ -446,6 +449,9 @@ See here the methods that you can override in your custom Files Pipeline:
       By default the :meth:`file_path` method returns
       ``full/<request URL hash>.<extension>``.
 
+      .. versionadded:: 2.4
+         The *item* parameter.
+
    .. method:: FilesPipeline.get_media_requests(item, info)
 
       As seen on the workflow, the pipeline will get the URLs of the images to
@@ -582,6 +588,9 @@ See here the methods that you can override in your custom Images Pipeline:
       By default the :meth:`file_path` method returns
       ``full/<request URL hash>.<extension>``.
 
+      .. versionadded:: 2.4
+         The *item* parameter.
+
    .. method:: ImagesPipeline.get_media_requests(item, info)
 
       Works the same way as :meth:`FilesPipeline.get_media_requests` method,
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 71331c841..912757850 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -102,7 +102,7 @@ module and documented in the :ref:`topics-settings-ref` section.
 Import paths and classes
 ========================
 
-.. versionadded:: VERSION
+.. versionadded:: 2.4.0
 
 When a setting references a callable object to be imported by Scrapy, such as a
 class or a function, there are two different ways you can specify that object:

From 47eac8374040c0dc389eb8152a60938dab358bc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sun, 11 Oct 2020 22:11:14 +0200
Subject: [PATCH 178/716] Set a release date for Scrapy 2.4.0

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index d5c342c86..a3889705d 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5,7 +5,7 @@ Release notes
 
 .. _release-2.4.0:
 
-Scrapy 2.4.0 (2020-10-??)
+Scrapy 2.4.0 (2020-10-11)
 -------------------------
 
 Highlights:

From c340e72988fc6ec615b7b9851c3d28c16c26a839 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sun, 11 Oct 2020 22:12:45 +0200
Subject: [PATCH 179/716] =?UTF-8?q?Bump=20version:=202.3.0=20=E2=86=92=202?=
 =?UTF-8?q?.4.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 3c1c8f891..0f142472e 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.3.0
+current_version = 2.4.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 276cbf9e2..197c4d5c2 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.3.0
+2.4.0

From fd663fd4ad5a69ba0403a2eec5bdc29a0109b0d4 Mon Sep 17 00:00:00 2001
From: GeorgeA92 <george.zatserklyany@gmail.com>
Date: Tue, 13 Oct 2020 18:35:06 +0300
Subject: [PATCH 180/716] __init__ stats parameter - optional, stats==None -
 covered.

---
 scrapy/downloadermiddlewares/httpcompression.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index ca80e9444..8980e9ca4 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -18,7 +18,7 @@ except ImportError:
 class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
-    def __init__(self, stats):
+    def __init__(self, stats=None):
         self.stats = stats
 
     @classmethod
@@ -40,8 +40,9 @@ class HttpCompressionMiddleware:
             if content_encoding:
                 encoding = content_encoding.pop()
                 decoded_body = self._decode(response.body, encoding.lower())
-                self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
-                self.stats.inc_value('httpcompression/response_count', spider=spider)
+                if self.stats:
+                    self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
+                    self.stats.inc_value('httpcompression/response_count', spider=spider)
                 respcls = responsetypes.from_args(
                     headers=response.headers, url=response.url, body=decoded_body
                 )

From d32d0d27393ce55490c44d9fb039130320461865 Mon Sep 17 00:00:00 2001
From: GeorgeA92 <george.zatserklyany@gmail.com>
Date: Tue, 13 Oct 2020 18:36:41 +0300
Subject: [PATCH 181/716] testcase added for HttpCompressionMiddleware with no
 stats

---
 tests/test_downloadermiddleware_httpcompression.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 144fd3378..5f8e76e0a 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -29,7 +29,7 @@ class HttpCompressionTest(TestCase):
     def setUp(self):
         self.crawler = get_crawler(Spider)
         self.spider = self.crawler._create_spider('scrapytest.org')
-        self.mw = HttpCompressionMiddleware(self.crawler.stats)
+        self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
         self.crawler.stats.open_spider(self.spider)
 
     def _getresponse(self, coding):
@@ -79,6 +79,18 @@ class HttpCompressionTest(TestCase):
         self.assertStatsEqual('httpcompression/response_count', 1)
         self.assertStatsEqual('httpcompression/response_bytes', 74837)
 
+    def test_process_response_gzip_no_stats(self):
+        mw = HttpCompressionMiddleware()
+        response = self._getresponse('gzip')
+        request = response.request
+
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        newresponse = mw.process_response(request, response, self.spider)
+        self.assertEqual(mw.stats, None)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b'<!DOCTYPE')
+        assert 'Content-Encoding' not in newresponse.headers
+
     def test_process_response_br(self):
         try:
             import brotli  # noqa: F401

From 872b2e4ce414c1268dcadcd8f3da7dab546e39f7 Mon Sep 17 00:00:00 2001
From: GeorgeA92 <george.zatserklyany@gmail.com>
Date: Tue, 13 Oct 2020 18:41:58 +0300
Subject: [PATCH 182/716] testcase added for COMPRESSION_ENABLED setting

---
 ...st_downloadermiddleware_httpcompression.py | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 5f8e76e0a..c2bcbd63c 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -6,6 +6,7 @@ from gzip import GzipFile
 from scrapy.spiders import Spider
 from scrapy.http import Response, Request, HtmlResponse
 from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS
+from scrapy.exceptions import NotConfigured
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.gz import gunzip
 from scrapy.utils.test import get_crawler
@@ -60,6 +61,27 @@ class HttpCompressionTest(TestCase):
             str(self.crawler.stats.get_stats(self.spider))
         )
 
+    def test_setting_false_compression_enabled(self):
+        self.assertRaises(
+            NotConfigured,
+            HttpCompressionMiddleware.from_crawler,
+            get_crawler(settings_dict={'COMPRESSION_ENABLED': False})
+        )
+
+    def test_setting_default_compression_enabled(self):
+        self.assertIsInstance(
+            HttpCompressionMiddleware.from_crawler(get_crawler()),
+            HttpCompressionMiddleware
+        )
+
+    def test_setting_true_compression_enabled(self):
+        self.assertIsInstance(
+            HttpCompressionMiddleware.from_crawler(
+                get_crawler(settings_dict={'COMPRESSION_ENABLED': True})
+            ),
+            HttpCompressionMiddleware
+        )
+
     def test_process_request(self):
         request = Request('http://scrapytest.org')
         assert 'Accept-Encoding' not in request.headers

From 585e4a8aee649f2b439c42d91d857ed1fbdf4fa5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 9 Oct 2020 10:41:19 -0300
Subject: [PATCH 183/716] Replace local server address

---
 tests/test_crawler.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 246e54860..b6de33189 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -330,7 +330,8 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
 
     def test_caching_hostname_resolver_finite_execution(self):
         with MockServer() as mock_server:
-            log = self.run_script("caching_hostname_resolver.py", mock_server.http_address)
+            http_address = mock_server.http_address.replace("0.0.0.0", "127.0.0.1")
+            log = self.run_script("caching_hostname_resolver.py", http_address)
             self.assertIn("Spider closed (finished)", log)
             self.assertNotIn("ERROR: Error downloading", log)
             self.assertNotIn("TimeoutError", log)

From 7187247c01d3629000b8b460eea26064043c5595 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 30 Oct 2020 10:23:59 +0100
Subject: [PATCH 184/716] Add PyDispatcher>=2.0.5 back to dependencies for old
 pip

---
 setup.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/setup.py b/setup.py
index c046684c0..aa75c7a34 100644
--- a/setup.py
+++ b/setup.py
@@ -33,12 +33,12 @@ install_requires = [
     'itemadapter>=0.1.0',
 ]
 extras_require = {}
-
+cpython_dependencies = [
+    'lxml>=3.5.0',
+    'PyDispatcher>=2.0.5',
+]
 if has_environment_marker_platform_impl_support():
-    extras_require[':platform_python_implementation == "CPython"'] = [
-        'lxml>=3.5.0',
-        'PyDispatcher>=2.0.5',
-    ]
+    extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies
     extras_require[':platform_python_implementation == "PyPy"'] = [
         # Earlier lxml versions are affected by
         # https://foss.heptapod.net/pypy/pypy/-/issues/2498,
@@ -49,7 +49,7 @@ if has_environment_marker_platform_impl_support():
         'PyPyDispatcher>=2.1.0',
     ]
 else:
-    install_requires.append('lxml>=3.5.0')
+    install_requires.extend(cpython_dependencies)
 
 
 setup(

From 13bcdc9f881727f5a140d4b8671c81c7de3bc572 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 30 Oct 2020 10:28:42 +0100
Subject: [PATCH 185/716] Restore pinned dependencies in tox.ini

---
 tox.ini | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/tox.ini b/tox.ini
index 5dc99621e..b90ed867f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -66,21 +66,17 @@ commands =
 [pinned]
 deps =
     -ctests/constraints.txt
-    #using cryptography-2.3.1 to solve dependencies conflict on pip20.2>=
-    cryptography==2.3.1
+    cryptography==2.0
     cssselect==0.9.1
     itemadapter==0.1.0
     parsel==1.5.0
     Protego==0.1.15
-    !pypy3: PyDispatcher==2.0.5
-    #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>=
-    pyOpenSSL==18.0.0
+    pyOpenSSL==16.2.0
     queuelib==1.4.2
     service_identity==16.0.0
-    Twisted==20.3.0
+    Twisted==17.9.0
     w3lib==1.17.0
-    #zope.interface==5.1.2 to resolve conflict with Twisted==20.3.0
-    zope.interface==5.1.2
+    zope.interface==4.1.3
     -rtests/requirements-py3.txt
     # Extras
     botocore==1.4.87
@@ -91,6 +87,7 @@ deps =
 deps =
     {[pinned]deps}
     lxml==3.5.0
+    PyDispatcher==2.0.5
     
 [testenv:windows-pinned]
 basepython = python3
@@ -99,6 +96,7 @@ deps =
     # First lxml version that includes a Windows wheel for Python 3.6, so we do
     # not need to build lxml from sources in a CI Windows job:
     lxml==3.8.0
+    PyDispatcher==2.0.5
 
 [testenv:extra-deps]
 deps =

From 3e5bc7773732aa7e9839934cc45cac8252a36aa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 30 Oct 2020 10:31:33 +0100
Subject: [PATCH 186/716] Remove pinning from the PyPy job

---
 tox.ini | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tox.ini b/tox.ini
index b90ed867f..878395e6f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -114,9 +114,6 @@ commands = {[testenv:asyncio]commands}
 
 [testenv:pypy3]
 basepython = pypy3
-deps = 
-    {[testenv]deps}
-    lxml==4.0.0
 commands =
     py.test {posargs:--durations=10 docs scrapy tests}
 

From a5872a0fad090c3d3f91f6e03a772265aa50f901 Mon Sep 17 00:00:00 2001
From: Georgiy Zatserklianyi <GeorgeA92@users.noreply.github.com>
Date: Fri, 30 Oct 2020 20:36:39 +0200
Subject: [PATCH 187/716] Fix output file overwrite with -O (FeedExporter
 updated) (#4859)

---
 scrapy/extensions/feedexport.py |  2 +-
 tests/test_commands.py          | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 7dcb2f52e..3fb4d0e2c 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -452,7 +452,7 @@ class FeedExporter:
         crawler = getattr(self, 'crawler', None)
 
         def build_instance(builder, *preargs):
-            return build_storage(builder, uri, preargs=preargs)
+            return build_storage(builder, uri, feed_options=feed_options, preargs=preargs)
 
         if crawler and hasattr(feedcls, 'from_crawler'):
             instance = build_instance(feedcls.from_crawler, crawler)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 3e54a0948..85aee55a5 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -680,9 +680,14 @@ class MySpider(scrapy.Spider):
         )
         return []
 """
+        with open(os.path.join(self.cwd, "example.json"), "w") as f1:
+            f1.write("not empty")
         args = ['-O', 'example.json']
         log = self.get_log(spider_code, args=args)
         self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
+        with open(os.path.join(self.cwd, "example.json")) as f2:
+            first_line = f2.readline()
+        self.assertNotEqual(first_line, "not empty")
 
     def test_output_and_overwrite_output(self):
         spider_code = """
@@ -813,9 +818,14 @@ class MySpider(scrapy.Spider):
         )
         return []
 """
+        with open(os.path.join(self.cwd, "example.json"), "w") as f1:
+            f1.write("not empty")
         args = ['-O', 'example.json']
         log = self.get_log(spider_code, args=args)
         self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
+        with open(os.path.join(self.cwd, "example.json")) as f2:
+            first_line = f2.readline()
+        self.assertNotEqual(first_line, "not empty")
 
     def test_output_and_overwrite_output(self):
         spider_code = """

From e9c3188189cffc965797b1b77fc5dc5cfa06b5cb Mon Sep 17 00:00:00 2001
From: Georgiy Zatserklianyi <GeorgeA92@users.noreply.github.com>
Date: Fri, 30 Oct 2020 21:23:29 +0200
Subject: [PATCH 188/716] Update
 scrapy/downloadermiddlewares/httpcompression.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/downloadermiddlewares/httpcompression.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 8980e9ca4..87f744956 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -25,7 +25,12 @@ class HttpCompressionMiddleware:
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('COMPRESSION_ENABLED'):
             raise NotConfigured
-        return cls(crawler.stats)
+        try:
+            return cls(stats=crawler.stats)
+        except TypeError:
+            result = cls()
+            result.stats = crawler.stats
+            return result
 
     def process_request(self, request, spider):
         request.headers.setdefault('Accept-Encoding',

From 7327145bf3d45b83b110a46ff79d2e337e36b520 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 30 Oct 2020 21:34:15 +0100
Subject: [PATCH 189/716] Remove mitmproxy from pinned environments

---
 tests/requirements-py3.txt  |  2 --
 tests/test_proxy_connect.py | 11 +++++------
 tox.ini                     | 10 +++++++++-
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 2247ed917..d44632d8b 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,8 +1,6 @@
 # Tests requirements
 attrs
 dataclasses; python_version == '3.6'
-mitmproxy; python_version >= '3.7'
-mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7'
 pyftpdlib
 # https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 9eabe6b49..0db148910 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -5,8 +5,6 @@ import re
 import sys
 from subprocess import Popen, PIPE
 from urllib.parse import urlsplit, urlunsplit
-from unittest import skipIf
-
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
@@ -57,13 +55,14 @@ def _wrong_credentials(proxy_url):
     return urlunsplit(bad_auth_proxy)
 
 
-@skipIf("pypy" in sys.executable,
-        "mitmproxy does not support PyPy")
-@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7),
-        "mitmproxy does not support Windows when running Python < 3.7")
 class ProxyConnectTestCase(TestCase):
 
     def setUp(self):
+        try:
+            import mitmproxy
+        except ImportError:
+            self.skipTest('mitmproxy is not installed')
+
         self.mockserver = MockServer()
         self.mockserver.__enter__()
         self._oldenv = os.environ.copy()
diff --git a/tox.ini b/tox.ini
index 878395e6f..6149a9c58 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,6 +11,10 @@ minversion = 1.7.0
 deps =
     -ctests/constraints.txt
     -rtests/requirements-py3.txt
+    # mitmproxy does not support PyPy
+    # mitmproxy does not support Windows when running Python < 3.7
+    mitmproxy; python_version >= '3.7' and implementation_name != 'pypy'
+    mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # Extras
     boto3>=1.13.0
     botocore>=1.4.87
@@ -26,7 +30,7 @@ download = true
 commands =
     py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
 install_command =
-    pip install --use-feature=2020-resolver {opts} {packages} 
+    pip install --use-feature=2020-resolver {opts} {packages}
 
 [testenv:typing]
 basepython = python3
@@ -78,6 +82,10 @@ deps =
     w3lib==1.17.0
     zope.interface==4.1.3
     -rtests/requirements-py3.txt
+
+    # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
+    # above, hence we do not install it in pinned environments at the moment
+
     # Extras
     botocore==1.4.87
     google-cloud-storage==1.29.0

From 8e7b756727cbd8c207acce3083ae29c85a6c6d51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 4 Nov 2020 21:26:55 +0100
Subject: [PATCH 190/716] Solve Flake8-reported issues

---
 tests/test_proxy_connect.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 0db148910..afdfb2578 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -1,6 +1,5 @@
 import json
 import os
-import platform
 import re
 import sys
 from subprocess import Popen, PIPE
@@ -59,7 +58,7 @@ class ProxyConnectTestCase(TestCase):
 
     def setUp(self):
         try:
-            import mitmproxy
+            import mitmproxy  # noqa: F401
         except ImportError:
             self.skipTest('mitmproxy is not installed')
 

From 906626cf0befe332cb61b9cdda7108d61afec776 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 4 Nov 2020 21:50:12 +0100
Subject: [PATCH 191/716] Skip MiddlewareUsingCoro::test_asyncdef on asyncio
 and old Twisted

---
 tests/test_downloadermiddleware.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 79f24c8a1..55af4c240 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,11 +1,12 @@
 import asyncio
-from unittest import mock
+from unittest import mock, SkipTest
 
 from pytest import mark
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
+from twisted.python.versions import Version
 
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
@@ -211,10 +212,24 @@ class MiddlewareUsingDeferreds(ManagerTestCase):
         self.assertFalse(download_func.called)
 
 
+@mark.usefixtures('reactor_pytest')
 class MiddlewareUsingCoro(ManagerTestCase):
     """Middlewares using asyncio coroutines should work"""
 
     def test_asyncdef(self):
+        import twisted
+        if (
+            self.reactor_pytest == 'asyncio'
+            and twisted.version < Version('twisted', 18, 4, 0)
+        ):
+            raise SkipTest(
+                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
+                'hangs when using AsyncIO and Twisted versions lower than '
+                '18.4.0'
+            )
+
+        from twisted.python.versions import Version
+
         resp = Response('http://example.com/index.html')
 
         class CoroMiddleware:

From 6eaf0c5cc99204daed601369c9cea6f7c5e2f878 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 4 Nov 2020 21:54:00 +0100
Subject: [PATCH 192/716] Use Ubuntu Bionic for PyPy tests to try to get a
 newer OpenSSL version recognized

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index b883c5b78..f0eafd137 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,6 +27,7 @@ matrix:
     - env: TOXENV=py
       python: 3.6
     - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
+      dist: bionic
 
     - env: TOXENV=py
       python: 3.7

From c292957cb19085137146bde72fe82639591c5e1c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Thu, 5 Nov 2020 11:15:58 -0300
Subject: [PATCH 193/716] Run Windows tests on GitHub actions (#4869)

---
 .github/workflows/main.yml | 31 +++++++++++++++++++++++++++++++
 azure-pipelines.yml        | 22 ----------------------
 2 files changed, 31 insertions(+), 22 deletions(-)
 create mode 100644 .github/workflows/main.yml
 delete mode 100644 azure-pipelines.yml

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
new file mode 100644
index 000000000..28771216c
--- /dev/null
+++ b/.github/workflows/main.yml
@@ -0,0 +1,31 @@
+name: Run test suite
+on: [push, pull_request]
+
+jobs:
+  test-windows:
+    name: "Windows Tests"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest]
+        python-version: [3.7, 3.8]
+        env: [TOXENV: py]
+        include:
+        - os: windows-latest
+          python-version: 3.6
+          env:
+            TOXENV: windows-pinned
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Run test suite
+      env: ${{ matrix.env }}
+      run: |
+        pip install -U tox twine wheel codecov
+        tox
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
deleted file mode 100644
index c03e258c7..000000000
--- a/azure-pipelines.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-variables:
-  TOXENV: py
-pool:
-  vmImage: 'windows-latest'
-strategy:
-  matrix:
-    Python36:
-      python.version: '3.6'
-      TOXENV: windows-pinned
-    Python37:
-      python.version: '3.7'
-    Python38:
-      python.version: '3.8'
-steps:
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: '$(python.version)'
-  displayName: 'Use Python $(python.version)'
-- script: |
-    pip install -U tox twine wheel codecov
-    tox
-  displayName: 'Run test suite'

From 5b5478ae9d6f8d5e0028fe47a63252679c457364 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 5 Nov 2020 14:01:34 -0300
Subject: [PATCH 194/716] Call asyncio.get_event_loop when installing the
 asyncio reactor

---
 scrapy/utils/reactor.py                       |  3 +-
 .../CrawlerProcess/asyncio_deferred_signal.py | 45 +++++++++++++++++++
 tests/test_crawler.py                         | 16 +++++++
 3 files changed, 63 insertions(+), 1 deletion(-)
 create mode 100644 tests/CrawlerProcess/asyncio_deferred_signal.py

diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 831d29462..6723d9b37 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -60,8 +60,9 @@ def install_reactor(reactor_path, event_loop_path=None):
             if event_loop_path is not None:
                 event_loop_class = load_object(event_loop_path)
                 event_loop = event_loop_class()
+                asyncio.set_event_loop(event_loop)
             else:
-                event_loop = asyncio.new_event_loop()
+                event_loop = asyncio.get_event_loop()
             asyncioreactor.install(eventloop=event_loop)
     else:
         *module, _ = reactor_path.split(".")
diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py
new file mode 100644
index 000000000..bce300afe
--- /dev/null
+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py
@@ -0,0 +1,45 @@
+import asyncio
+import sys
+
+import scrapy
+
+from scrapy.crawler import CrawlerProcess
+from twisted.internet.defer import Deferred
+
+
+class UppercasePipeline:
+    async def _open_spider(self, spider):
+        spider.logger.info("async pipeline opened!")
+        await asyncio.sleep(0.1)
+
+    def open_spider(self, spider):
+        loop = asyncio.get_event_loop()
+        return Deferred.fromFuture(loop.create_task(self._open_spider(spider)))
+
+    def process_item(self, item, spider):
+        return {"url": item["url"].upper()}
+
+
+class UrlSpider(scrapy.Spider):
+    name = "url_spider"
+    start_urls = ["data:,"]
+    custom_settings = {
+        "ITEM_PIPELINES": {UppercasePipeline: 100},
+    }
+
+    def parse(self, response):
+        yield {"url": response.url}
+
+
+if __name__ == "__main__":
+    try:
+        ASYNCIO_EVENT_LOOP = sys.argv[1]
+    except IndexError:
+        ASYNCIO_EVENT_LOOP = None
+
+    process = CrawlerProcess(settings={
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+        "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP,
+    })
+    process.crawl(UrlSpider)
+    process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index b6de33189..0faaa79a3 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -364,6 +364,22 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
 
+    @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly")
+    @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows")
+    def test_custom_loop_asyncio_deferred_signal(self):
+        log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn("Using asyncio event loop: uvloop.Loop", log)
+        self.assertIn("async pipeline opened!", log)
+
+    def test_default_loop_asyncio_deferred_signal(self):
+        log = self.run_script("asyncio_deferred_signal.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertNotIn("Using asyncio event loop: uvloop.Loop", log)
+        self.assertIn("async pipeline opened!", log)
+
 
 class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
     script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')

From ea851b910e580e840653274182548314e15b4312 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 6 Nov 2020 12:34:29 +0100
Subject: [PATCH 195/716] Clean up Twisted version check

---
 tests/test_downloadermiddleware.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 55af4c240..6168e0176 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -2,6 +2,7 @@ import asyncio
 from unittest import mock, SkipTest
 
 from pytest import mark
+from twisted import version as twisted_version
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
@@ -217,10 +218,9 @@ class MiddlewareUsingCoro(ManagerTestCase):
     """Middlewares using asyncio coroutines should work"""
 
     def test_asyncdef(self):
-        import twisted
         if (
             self.reactor_pytest == 'asyncio'
-            and twisted.version < Version('twisted', 18, 4, 0)
+            and twisted_version < Version('twisted', 18, 4, 0)
         ):
             raise SkipTest(
                 'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
@@ -228,8 +228,6 @@ class MiddlewareUsingCoro(ManagerTestCase):
                 '18.4.0'
             )
 
-        from twisted.python.versions import Version
-
         resp = Response('http://example.com/index.html')
 
         class CoroMiddleware:

From fea5a118993855808183b32809c2c304db28c364 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 6 Nov 2020 12:59:46 +0100
Subject: [PATCH 196/716] Also skip test_asyncdef_asyncio on old Twisted
 versions

---
 tests/test_downloadermiddleware.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 6168e0176..b538a0ed3 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -248,6 +248,12 @@ class MiddlewareUsingCoro(ManagerTestCase):
 
     @mark.only_asyncio()
     def test_asyncdef_asyncio(self):
+        if twisted_version < Version('twisted', 18, 4, 0):
+            raise SkipTest(
+                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
+                'hangs when using Twisted versions lower than 18.4.0'
+            )
+
         resp = Response('http://example.com/index.html')
 
         class CoroMiddleware:

From a3e53027ec35498dcb931404e02689877da6aeb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 6 Nov 2020 14:16:26 +0100
Subject: [PATCH 197/716] Test HttpCompressionMiddleware subclasses with
 custom, parameterless __init__

---
 .../downloadermiddlewares/httpcompression.py  | 12 +++++--
 ...st_downloadermiddleware_httpcompression.py | 36 ++++++++++++++++---
 2 files changed, 42 insertions(+), 6 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 87f744956..1808154d2 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,9 +1,11 @@
+import warnings
 import zlib
 
-from scrapy.utils.gz import gunzip
+from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
-from scrapy.exceptions import NotConfigured
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.gz import gunzip
 
 
 ACCEPTED_ENCODINGS = [b'gzip', b'deflate']
@@ -28,6 +30,12 @@ class HttpCompressionMiddleware:
         try:
             return cls(stats=crawler.stats)
         except TypeError:
+            warnings.warn(
+                "HttpCompressionMiddleware subclasses must either modify "
+                "their '__init__' method to support a 'stats' parameter or "
+                "reimplement the 'from_crawler' method.",
+                ScrapyDeprecationWarning,
+            )
             result = cls()
             result.stats = crawler.stats
             return result
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index c2bcbd63c..2ee87aa67 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -1,12 +1,13 @@
-from io import BytesIO
-from unittest import TestCase, SkipTest
-from os.path import join
 from gzip import GzipFile
+from io import BytesIO
+from os.path import join
+from unittest import TestCase, SkipTest
+from warnings import catch_warnings
 
 from scrapy.spiders import Spider
 from scrapy.http import Response, Request, HtmlResponse
 from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.gz import gunzip
 from scrapy.utils.test import get_crawler
@@ -321,3 +322,30 @@ class HttpCompressionTest(TestCase):
         self.assertEqual(response.body, b'')
         self.assertStatsEqual('httpcompression/response_count', None)
         self.assertStatsEqual('httpcompression/response_bytes', None)
+
+
+class HttpCompressionSubclassTest(TestCase):
+
+    def test_init_missing_stats(self):
+        class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
+
+            def __init__(self):
+                super().__init__()
+
+        crawler = get_crawler(Spider)
+        with catch_warnings(record=True) as caught_warnings:
+            instance = HttpCompressionMiddlewareSubclass.from_crawler(crawler)
+        messages = tuple(
+            str(warning.message) for warning in caught_warnings
+            if warning.category is ScrapyDeprecationWarning
+        )
+        self.assertEqual(
+            messages,
+            (
+                (
+                    "HttpCompressionMiddleware subclasses must either modify "
+                    "their '__init__' method to support a 'stats' parameter "
+                    "or reimplement the 'from_crawler' method."
+                ),
+            )
+        )

From 3095d39740c4818d2c1c98d392255981ebed2a10 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 6 Nov 2020 12:16:10 -0300
Subject: [PATCH 198/716] Test: disable asyncio reactor on Windows for Py>=3.8

---
 tests/test_crawler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 0faaa79a3..ab113710d 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -373,6 +373,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
         self.assertIn("async pipeline opened!", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_default_loop_asyncio_deferred_signal(self):
         log = self.run_script("asyncio_deferred_signal.py")
         self.assertIn("Spider closed (finished)", log)

From 1941f607ca54694d4822933eba973665baa1b45b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 6 Nov 2020 16:25:56 +0100
Subject: [PATCH 199/716] Skip 2 additional tests with older Twisted versions

---
 tests/test_utils_signal.py | 33 ++++++++++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index b66588efb..18a8fb49c 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -1,11 +1,14 @@
 import asyncio
+from unittest import SkipTest
 
+from pydispatch import dispatcher
 from pytest import mark
 from testfixtures import LogCapture
-from twisted.trial import unittest
-from twisted.python.failure import Failure
+from twisted import version as twisted_version
 from twisted.internet import defer, reactor
-from pydispatch import dispatcher
+from twisted.python.failure import Failure
+from twisted.python.versions import Version
+from twisted.trial import unittest
 
 from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
 from scrapy.utils.test import get_from_asyncio_queue
@@ -68,6 +71,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):
         return d
 
 
+@mark.usefixtures('reactor_pytest')
 class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
 
     async def ok_handler(self, arg, handlers_called):
@@ -76,6 +80,19 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
         await defer.succeed(42)
         return "OK"
 
+    def test_send_catch_log(self):
+        if (
+            self.reactor_pytest == 'asyncio'
+            and twisted_version < Version('twisted', 18, 4, 0)
+        ):
+            raise SkipTest(
+                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
+                'fails due to a timeout when using AsyncIO and Twisted '
+                'versions lower than 18.4.0'
+            )
+
+        return super().test_send_catch_log()
+
 
 @mark.only_asyncio()
 class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
@@ -86,6 +103,16 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
         await asyncio.sleep(0.2)
         return await get_from_asyncio_queue("OK")
 
+    def test_send_catch_log(self):
+        if (twisted_version < Version('twisted', 18, 4, 0):
+            raise SkipTest(
+                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
+                'fails due to a timeout when using Twisted versions lower '
+                'than 18.4.0'
+            )
+
+        return super().test_send_catch_log()
+
 
 class SendCatchLogTest2(unittest.TestCase):
 

From ee98771fa72aa5e109292af5d734e5c49bae64e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 6 Nov 2020 16:42:32 +0100
Subject: [PATCH 200/716] Remove unused variable

---
 tests/test_downloadermiddleware_httpcompression.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 2ee87aa67..38d8534ca 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -334,7 +334,7 @@ class HttpCompressionSubclassTest(TestCase):
 
         crawler = get_crawler(Spider)
         with catch_warnings(record=True) as caught_warnings:
-            instance = HttpCompressionMiddlewareSubclass.from_crawler(crawler)
+            HttpCompressionMiddlewareSubclass.from_crawler(crawler)
         messages = tuple(
             str(warning.message) for warning in caught_warnings
             if warning.category is ScrapyDeprecationWarning

From 4b28da433384f67d0e6c7d4d557f5bcf268ae846 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 6 Nov 2020 16:46:22 +0100
Subject: [PATCH 201/716] Fix syntax error

---
 tests/test_utils_signal.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index 18a8fb49c..ad7394232 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -104,7 +104,7 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
         return await get_from_asyncio_queue("OK")
 
     def test_send_catch_log(self):
-        if (twisted_version < Version('twisted', 18, 4, 0):
+        if twisted_version < Version('twisted', 18, 4, 0):
             raise SkipTest(
                 'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
                 'fails due to a timeout when using Twisted versions lower '

From 114229eb4a5e3e0289000500cf063518be908d40 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 6 Nov 2020 13:29:14 -0300
Subject: [PATCH 202/716] Docs: add a note about asyncio.set_event_loop

---
 docs/topics/settings.rst | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 912757850..0086a6c74 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -249,19 +249,25 @@ ASYNCIO_EVENT_LOOP
 
 Default: ``None``
 
-Import path of a given asyncio event loop class.
+Import path of a given ``asyncio`` event loop class.
 
-If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the 
-asyncio event loop to be used with it. Set the setting to the import path of the 
+If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
+asyncio event loop to be used with it. Set the setting to the import path of the
 desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
 event loop will be used.
 
 If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
-function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop 
-class to be used.  
+function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
+class to be used.
 
 Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
 
+.. caution:: Please be aware that, when using a non-default event loop
+    (either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
+    :func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
+    :func:`asyncio.set_event_loop`, which will set the specified event loop
+    as the current loop for the current OS thread.
+
 .. setting:: BOT_NAME
 
 BOT_NAME

From a2c4a7f9200a06b227a297b9dd2919fe3ec37dbe Mon Sep 17 00:00:00 2001
From: Valdir Stumm Junior <stummjr@gmail.com>
Date: Sun, 8 Nov 2020 19:12:18 -0300
Subject: [PATCH 203/716] Add missing f-string prefix to genspider output

---
 scrapy/commands/genspider.py | 2 +-
 tests/test_commands.py       | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 72248bded..5f44daa70 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -98,7 +98,7 @@ class Command(ScrapyCommand):
         print(f"Created spider {name!r} using template {template_name!r} ",
               end=('' if spiders_module else '\n'))
         if spiders_module:
-            print("in module:\n  {spiders_module.__name__}.{module}")
+            print(f"in module:\n  {spiders_module.__name__}.{module}")
 
     def _find_template(self, template):
         template_file = join(self.templates_dir, f'{template}.tmpl')
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 85aee55a5..d3ac05eac 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -389,8 +389,9 @@ class GenspiderCommandTest(CommandTest):
     def test_template(self, tplname='crawl'):
         args = [f'--template={tplname}'] if tplname else []
         spname = 'test_spider'
+        spmodule = f"{self.project_name}.spiders.{spname}"
         p, out, err = self.proc('genspider', spname, 'test.com', *args)
-        self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module", out)
+        self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep}  {spmodule}", out)
         self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
         modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
         p, out, err = self.proc('genspider', spname, 'test.com', *args)

From 7e98a76ac455a8c69950104766719cde313bbb74 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 9 Nov 2020 12:17:15 -0300
Subject: [PATCH 204/716] Use deferred_from_coro in asyncio test

---
 tests/CrawlerProcess/asyncio_deferred_signal.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py
index bce300afe..dd82aa2ff 100644
--- a/tests/CrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py
@@ -1,9 +1,9 @@
 import asyncio
 import sys
 
-import scrapy
-
+from scrapy import Spider
 from scrapy.crawler import CrawlerProcess
+from scrapy.utils.defer import deferred_from_coro
 from twisted.internet.defer import Deferred
 
 
@@ -14,13 +14,13 @@ class UppercasePipeline:
 
     def open_spider(self, spider):
         loop = asyncio.get_event_loop()
-        return Deferred.fromFuture(loop.create_task(self._open_spider(spider)))
+        return deferred_from_coro(self._open_spider(spider))
 
     def process_item(self, item, spider):
         return {"url": item["url"].upper()}
 
 
-class UrlSpider(scrapy.Spider):
+class UrlSpider(Spider):
     name = "url_spider"
     start_urls = ["data:,"]
     custom_settings = {

From b20cfef1e54d9f22769f6d0ec6ae06031bf86ec3 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 9 Nov 2020 13:58:52 -0300
Subject: [PATCH 205/716] Remove unnecessary line from test

---
 tests/CrawlerProcess/asyncio_deferred_signal.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py
index dd82aa2ff..46c2a12a4 100644
--- a/tests/CrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py
@@ -13,7 +13,6 @@ class UppercasePipeline:
         await asyncio.sleep(0.1)
 
     def open_spider(self, spider):
-        loop = asyncio.get_event_loop()
         return deferred_from_coro(self._open_spider(spider))
 
     def process_item(self, item, spider):

From c20b34269f488dae4de9433d9c7c783bc481bc6f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Tue, 10 Nov 2020 15:35:09 -0300
Subject: [PATCH 206/716] Remove unnecessary pytest-azurepipelines package
 (#4876)

---
 tests/requirements-py3.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 2eed2f5da..7f8a5c52e 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -6,7 +6,6 @@ mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7'
 pyftpdlib
 # https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1
-pytest-azurepipelines
 pytest-cov
 pytest-twisted >= 1.11
 pytest-xdist

From 99cc853d6953d336ca65e0eecc0cb3286306bacf Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 29 Sep 2020 23:53:37 -0300
Subject: [PATCH 207/716] Response.protocol attribute

---
 scrapy/core/downloader/handlers/http11.py |  1 +
 scrapy/http/response/__init__.py          | 17 ++++++++++++++---
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 1b041c8a8..0f30b01f9 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -442,6 +442,7 @@ class ScrapyAgent:
             flags=result["flags"],
             certificate=result["certificate"],
             ip_address=result["ip_address"],
+            protocol=getattr(result["txresponse"], "version", None),
         )
         if result.get("failure"):
             result["failure"].value.response = response
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index c635fde69..f4ef79c72 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -17,8 +17,18 @@ from scrapy.utils.trackref import object_ref
 
 class Response(object_ref):
 
-    def __init__(self, url, status=200, headers=None, body=b'', flags=None,
-                 request=None, certificate=None, ip_address=None):
+    def __init__(
+        self,
+        url,
+        status=200,
+        headers=None,
+        body=b"",
+        flags=None,
+        request=None,
+        certificate=None,
+        ip_address=None,
+        protocol=None,
+    ):
         self.headers = Headers(headers or {})
         self.status = int(status)
         self._set_body(body)
@@ -27,6 +37,7 @@ class Response(object_ref):
         self.flags = [] if flags is None else list(flags)
         self.certificate = certificate
         self.ip_address = ip_address
+        self.protocol = protocol
 
     @property
     def cb_kwargs(self):
@@ -90,7 +101,7 @@ class Response(object_ref):
         given new values.
         """
         for x in ['url', 'status', 'headers', 'body',
-                  'request', 'flags', 'certificate', 'ip_address']:
+                  'request', 'flags', 'certificate', 'ip_address', 'protocol']:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)

From 5b6b56240c24d02ef69e6cc591ffb2529bc3f36a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Nov 2020 01:08:37 -0300
Subject: [PATCH 208/716] Test Response.protocol attribute

---
 tests/test_downloader_handlers.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 3e8d7e6b9..eb6d40df7 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -489,6 +489,13 @@ class Http11TestCase(HttpTestCase):
     def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
         return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked')
 
+    def test_protocol(self):
+        request = Request(self.getURL("host"), method="GET")
+        d = self.download_request(request, Spider("foo"))
+        d.addCallback(lambda r: r.protocol)
+        d.addCallback(self.assertEqual, (b"HTTP", 1, 1))
+        return d
+
 
 class Https11TestCase(Http11TestCase):
     scheme = 'https'

From 587b4dd71fca12fa5fcc766b891540af77cb27c8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Nov 2020 01:20:50 -0300
Subject: [PATCH 209/716] Docs for the Response.protocol attribute

---
 docs/topics/request-response.rst | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index f3aaa2c8f..d7d5cd44e 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -693,9 +693,22 @@ Response objects
     :param ip_address: The IP address of the server from which the Response originated.
     :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
 
+    :param protocol: A tuple containing information about the protocol that was used
+        to download the response. Taken from the ``version`` attribute of the
+        corresponding :class:`twisted.web.client.Response` object, it will tipically
+        consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)``
+        to represent "HTTP/1.1".
+    :type protocol: :class:`tuple`
+
+    .. versionadded:: 2.X.X
+       The ``protocol`` parameter.
+
     .. versionadded:: 2.1.0
        The ``ip_address`` parameter.
 
+    .. versionadded:: 2.0.0
+       The ``certificate`` parameter.
+
     .. attribute:: Response.url
 
         A string containing the URL of the response.
@@ -780,6 +793,8 @@ Response objects
 
     .. attribute:: Response.certificate
 
+        .. versionadded:: 2.0.0
+
         A :class:`twisted.internet.ssl.Certificate` object representing
         the server's SSL certificate.
 
@@ -795,6 +810,20 @@ Response objects
         handler, i.e. for ``http(s)`` responses. For other handlers,
         :attr:`ip_address` is always ``None``.
 
+    .. attribute:: Response.protocol
+
+        .. versionadded:: 2.X.X
+
+        A tuple containing information about the protocol that was used
+        to download the response. Taken from the ``version`` attribute of the
+        corresponding :class:`twisted.web.client.Response` object, it will tipically
+        consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)``
+        to represent "HTTP/1.1".
+
+        This attribute is currently only populated by the HTTP 1.1 download
+        handler, i.e. for ``http(s)`` responses. For other handlers,
+        :attr:`protocol` is always ``None``.
+
     .. method:: Response.copy()
 
        Returns a new Response which is a copy of this Response.

From 0fb7bcb2cf1606f63f1863bea254a34386c6a0f1 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Nov 2020 01:26:03 -0300
Subject: [PATCH 210/716] Style adjustment

---
 scrapy/http/response/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index f4ef79c72..185a9bb67 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -100,8 +100,9 @@ class Response(object_ref):
         """Create a new Response with the same attributes except for those
         given new values.
         """
-        for x in ['url', 'status', 'headers', 'body',
-                  'request', 'flags', 'certificate', 'ip_address', 'protocol']:
+        for x in [
+            "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol",
+        ]:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)

From 61d089485c7ba66649b936e34833b2013fa12458 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Nov 2020 01:31:15 -0300
Subject: [PATCH 211/716] Docs: sort versionadded directives

---
 docs/topics/request-response.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index d7d5cd44e..f1be41dde 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -700,14 +700,14 @@ Response objects
         to represent "HTTP/1.1".
     :type protocol: :class:`tuple`
 
-    .. versionadded:: 2.X.X
-       The ``protocol`` parameter.
+    .. versionadded:: 2.0.0
+       The ``certificate`` parameter.
 
     .. versionadded:: 2.1.0
        The ``ip_address`` parameter.
 
-    .. versionadded:: 2.0.0
-       The ``certificate`` parameter.
+    .. versionadded:: 2.X.X
+       The ``protocol`` parameter.
 
     .. attribute:: Response.url
 

From 22424125560496c9d131c9b7226aaf0f794e5ad8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Nov 2020 10:50:54 -0300
Subject: [PATCH 212/716] Docs: placeholder for versionadded directive

---
 docs/topics/request-response.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index f1be41dde..1cb724227 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -706,7 +706,7 @@ Response objects
     .. versionadded:: 2.1.0
        The ``ip_address`` parameter.
 
-    .. versionadded:: 2.X.X
+    .. versionadded:: VERSION
        The ``protocol`` parameter.
 
     .. attribute:: Response.url
@@ -812,7 +812,7 @@ Response objects
 
     .. attribute:: Response.protocol
 
-        .. versionadded:: 2.X.X
+        .. versionadded:: VERSION
 
         A tuple containing information about the protocol that was used
         to download the response. Taken from the ``version`` attribute of the

From 5e9a99e6a1e940864d9157252592f04658eaf851 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Nov 2020 11:15:29 -0300
Subject: [PATCH 213/716] Reponse.protocol as string

---
 docs/topics/request-response.rst          | 20 +++++++-------------
 scrapy/core/downloader/handlers/http11.py |  7 ++++++-
 scrapy/core/downloader/webclient.py       |  4 ++--
 tests/test_downloader_handlers.py         |  9 ++++++++-
 4 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 1cb724227..98906992d 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -693,12 +693,9 @@ Response objects
     :param ip_address: The IP address of the server from which the Response originated.
     :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
 
-    :param protocol: A tuple containing information about the protocol that was used
-        to download the response. Taken from the ``version`` attribute of the
-        corresponding :class:`twisted.web.client.Response` object, it will tipically
-        consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)``
-        to represent "HTTP/1.1".
-    :type protocol: :class:`tuple`
+    :param protocol: The protocol that was used to download the response.
+        For instance: "HTTP/1.0", "HTTP/1.1"
+    :type protocol: :class:`str`
 
     .. versionadded:: 2.0.0
        The ``certificate`` parameter.
@@ -814,14 +811,11 @@ Response objects
 
         .. versionadded:: VERSION
 
-        A tuple containing information about the protocol that was used
-        to download the response. Taken from the ``version`` attribute of the
-        corresponding :class:`twisted.web.client.Response` object, it will tipically
-        consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)``
-        to represent "HTTP/1.1".
+        The protocol that was used to download the response.
+        For instance: "HTTP/1.0", "HTTP/1.1"
 
-        This attribute is currently only populated by the HTTP 1.1 download
-        handler, i.e. for ``http(s)`` responses. For other handlers,
+        This attribute is currently only populated by the HTTP download
+        handlers, i.e. for ``http(s)`` responses. For other handlers,
         :attr:`protocol` is always ``None``.
 
     .. method:: Response.copy()
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 0f30b01f9..c7553eb87 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -434,6 +434,11 @@ class ScrapyAgent:
     def _cb_bodydone(self, result, request, url):
         headers = Headers(result["txresponse"].headers.getAllRawHeaders())
         respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
+        try:
+            version = result["txresponse"].version
+            protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}"
+        except (AttributeError, TypeError):
+            protocol = None
         response = respcls(
             url=url,
             status=int(result["txresponse"].code),
@@ -442,7 +447,7 @@ class ScrapyAgent:
             flags=result["flags"],
             certificate=result["certificate"],
             ip_address=result["ip_address"],
-            protocol=getattr(result["txresponse"], "version", None),
+            protocol=protocol,
         )
         if result.get("failure"):
             result["failure"].value.response = response
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index c13683393..9524cce2b 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -7,7 +7,7 @@ from twisted.internet.protocol import ClientFactory
 
 from scrapy.http import Headers
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_bytes
+from scrapy.utils.python import to_bytes, to_unicode
 from scrapy.responsetypes import responsetypes
 
 
@@ -110,7 +110,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
         status = int(self.status)
         headers = Headers(self.response_headers)
         respcls = responsetypes.from_args(headers=headers, url=self._url)
-        return respcls(url=self._url, status=status, headers=headers, body=body)
+        return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
 
     def _set_connection_attributes(self, request):
         parsed = urlparse_cached(request)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index eb6d40df7..a8763a7a5 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -360,6 +360,13 @@ class Http10TestCase(HttpTestCase):
     """HTTP 1.0 test case"""
     download_handler_cls = HTTP10DownloadHandler
 
+    def test_protocol(self):
+        request = Request(self.getURL("host"), method="GET")
+        d = self.download_request(request, Spider("foo"))
+        d.addCallback(lambda r: r.protocol)
+        d.addCallback(self.assertEqual, "HTTP/1.0")
+        return d
+
 
 class Https10TestCase(Http10TestCase):
     scheme = 'https'
@@ -493,7 +500,7 @@ class Http11TestCase(HttpTestCase):
         request = Request(self.getURL("host"), method="GET")
         d = self.download_request(request, Spider("foo"))
         d.addCallback(lambda r: r.protocol)
-        d.addCallback(self.assertEqual, (b"HTTP", 1, 1))
+        d.addCallback(self.assertEqual, "HTTP/1.1")
         return d
 
 

From b0368228d7f6391c0df41fca1609c6548613ad6b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 11 Nov 2020 11:18:03 -0300
Subject: [PATCH 214/716] Add exception to catch

---
 scrapy/core/downloader/handlers/http11.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index c7553eb87..a0fd837b1 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -437,7 +437,7 @@ class ScrapyAgent:
         try:
             version = result["txresponse"].version
             protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}"
-        except (AttributeError, TypeError):
+        except (AttributeError, TypeError, IndexError):
             protocol = None
         response = respcls(
             url=url,

From 034d61e6cb8391f045eb9b20824964e951549e01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 11 Nov 2020 16:46:03 +0100
Subject: [PATCH 215/716] =?UTF-8?q?Restrict=20pip=E2=80=99s=20--use-featur?=
 =?UTF-8?q?e=3D2020-resolver=20to=20the=20extra-deps=20environment?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index 6149a9c58..f35466d05 100644
--- a/tox.ini
+++ b/tox.ini
@@ -29,8 +29,6 @@ passenv =
 download = true 
 commands =
     py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
-install_command =
-    pip install --use-feature=2020-resolver {opts} {packages}
 
 [testenv:typing]
 basepython = python3
@@ -111,6 +109,8 @@ deps =
     {[testenv]deps}
     reppy
     robotexclusionrulesparser
+install_command =
+    pip install --use-feature=2020-resolver {opts} {packages}
 
 [testenv:asyncio]
 commands =

From 85604e1078b927c2a875040e29c58b4594c8d386 Mon Sep 17 00:00:00 2001
From: joaquin garmendia <joaquingc123@gmail.com>
Date: Wed, 11 Nov 2020 15:16:01 -0500
Subject: [PATCH 216/716] Add failed and success count stats to feedstorage
 backends (#4850)

---
 scrapy/extensions/feedexport.py |  22 ++++---
 tests/test_feedexport.py        | 106 ++++++++++++++++++++++++++++----
 2 files changed, 110 insertions(+), 18 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 3fb4d0e2c..bec114707 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -319,18 +319,26 @@ class FeedExporter:
         # Use `largs=log_args` to copy log_args into function's scope
         # instead of using `log_args` from the outer scope
         d.addCallback(
-            lambda _, largs=log_args: logger.info(
-                logfmt % "Stored", largs, extra={'spider': spider}
-            )
+            self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__
         )
         d.addErrback(
-            lambda f, largs=log_args: logger.error(
-                logfmt % "Error storing", largs,
-                exc_info=failure_to_exc_info(f), extra={'spider': spider}
-            )
+            self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__
         )
         return d
 
+    def _handle_store_error(self, f, largs, logfmt, spider, slot_type):
+        logger.error(
+            logfmt % "Error storing", largs,
+            exc_info=failure_to_exc_info(f), extra={'spider': spider}
+        )
+        self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
+
+    def _handle_store_success(self, f, largs, logfmt, spider, slot_type):
+        logger.info(
+            logfmt % "Stored", largs, extra={'spider': spider}
+        )
+        self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
+
     def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
         """
         Redirect the output data stream to a new file.
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 1ea4e8e12..d248824fc 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -8,6 +8,7 @@ import tempfile
 import warnings
 from abc import ABC, abstractmethod
 from collections import defaultdict
+from contextlib import ExitStack
 from io import BytesIO
 from logging import getLogger
 from pathlib import Path
@@ -47,6 +48,21 @@ from scrapy.utils.test import (
 )
 
 from tests.mockserver import MockFTPServer, MockServer
+from tests.spiders import ItemSpider
+
+
+def path_to_url(path):
+    return urljoin('file:', pathname2url(str(path)))
+
+
+def printf_escape(string):
+    return string.replace('%', '%%')
+
+
+def build_url(path):
+    if path[0] != '/':
+        path = '/' + path
+    return urljoin('file:', path)
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -620,12 +636,6 @@ class FeedExportTest(FeedExportTestBase):
     def run_and_export(self, spider_cls, settings):
         """ Run spider with specified settings; return exported data. """
 
-        def path_to_url(path):
-            return urljoin('file:', pathname2url(str(path)))
-
-        def printf_escape(string):
-            return string.replace('%', '%%')
-
         FEEDS = settings.get('FEEDS') or {}
         settings['FEEDS'] = {
             printf_escape(path_to_url(file_path)): feed_options
@@ -748,6 +758,69 @@ class FeedExportTest(FeedExportTestBase):
         result = self._load_until_eof(data['marshal'], load_func=marshal.load)
         self.assertEqual(expected, result)
 
+    @defer.inlineCallbacks
+    def test_stats_file_success(self):
+        settings = {
+            "FEEDS": {
+                printf_escape(path_to_url(self._random_temp_filename())): {
+                    "format": "json",
+                }
+            },
+        }
+        crawler = get_crawler(ItemSpider, settings)
+        with MockServer() as mockserver:
+            yield crawler.crawl(mockserver=mockserver)
+        self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())
+        self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1)
+
+    @defer.inlineCallbacks
+    def test_stats_file_failed(self):
+        settings = {
+            "FEEDS": {
+                printf_escape(path_to_url(self._random_temp_filename())): {
+                    "format": "json",
+                }
+            },
+        }
+        crawler = get_crawler(ItemSpider, settings)
+        with ExitStack() as stack:
+            mockserver = stack.enter_context(MockServer())
+            stack.enter_context(
+                mock.patch(
+                    "scrapy.extensions.feedexport.FileFeedStorage.store",
+                    side_effect=KeyError("foo"))
+            )
+            yield crawler.crawl(mockserver=mockserver)
+        self.assertIn("feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats())
+        self.assertEqual(crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1)
+
+    @defer.inlineCallbacks
+    def test_stats_multiple_file(self):
+        settings = {
+            'AWS_ACCESS_KEY_ID': 'access_key',
+            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+            "FEEDS": {
+                printf_escape(path_to_url(self._random_temp_filename())): {
+                    "format": "json",
+                },
+                "s3://bucket/key/foo.csv": {
+                    "format": "csv",
+                },
+                "stdout:": {
+                    "format": "xml",
+                }
+            },
+        }
+        crawler = get_crawler(ItemSpider, settings)
+        with MockServer() as mockserver, mock.patch.object(S3FeedStorage, "store"):
+            yield crawler.crawl(mockserver=mockserver)
+        self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())
+        self.assertIn("feedexport/success_count/S3FeedStorage", crawler.stats.get_stats())
+        self.assertIn("feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats())
+        self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1)
+        self.assertEqual(crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1)
+        self.assertEqual(crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1)
+
     @defer.inlineCallbacks
     def test_export_items(self):
         # feed exporters use field names from Item
@@ -1256,11 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
     def run_and_export(self, spider_cls, settings):
         """ Run spider with specified settings; return exported data. """
 
-        def build_url(path):
-            if path[0] != '/':
-                path = '/' + path
-            return urljoin('file:', path)
-
         FEEDS = settings.get('FEEDS') or {}
         settings['FEEDS'] = {
             build_url(file_path): feed
@@ -1550,6 +1618,22 @@ class BatchDeliveriesTest(FeedExportTestBase):
         data = yield self.exported_data(items, settings)
         self.assertEqual(len(items) + 1, len(data['json']))
 
+    @defer.inlineCallbacks
+    def test_stats_batch_file_success(self):
+        settings = {
+            "FEEDS": {
+                build_url(os.path.join(self._random_temp_filename(), "json", self._file_mark)): {
+                    "format": "json",
+                }
+            },
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 1,
+        }
+        crawler = get_crawler(ItemSpider, settings)
+        with MockServer() as mockserver:
+            yield crawler.crawl(total=2, mockserver=mockserver)
+        self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())
+        self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12)
+
     @defer.inlineCallbacks
     def test_s3_export(self):
         skip_if_no_boto()

From 2405df49f14cbc052d73e58a819a87417b2502e8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 16 Nov 2020 12:50:33 -0300
Subject: [PATCH 217/716] Add tests for Response.protocol=None

---
 tests/test_downloader_handlers.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index a8763a7a5..f51a6cd3c 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -115,6 +115,7 @@ class FileTestCase(unittest.TestCase):
             self.assertEqual(response.url, request.url)
             self.assertEqual(response.status, 200)
             self.assertEqual(response.body, b'0123456789')
+            self.assertEqual(response.protocol, None)
 
         request = Request(path_to_file_uri(self.tmpname + '^'))
         assert request.url.upper().endswith('%5E')
@@ -976,6 +977,7 @@ class BaseFTPTestCase(unittest.TestCase):
             self.assertEqual(r.status, 200)
             self.assertEqual(r.body, b'I have the power!')
             self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']})
+            self.assertIsNone(r.protocol)
         return self._add_test_callbacks(d, _test)
 
     def test_ftp_download_path_with_spaces(self):
@@ -1134,3 +1136,10 @@ class DataURITestCase(unittest.TestCase):
 
         request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D')
         return self.download_request(request, self.spider).addCallback(_test)
+
+    def test_protocol(self):
+        def _test(response):
+            self.assertIsNone(response.protocol)
+
+        request = Request("data:,")
+        return self.download_request(request, self.spider).addCallback(_test)

From 15d301e968aa3e26a28f771cf1b45635e84ef094 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 17 Nov 2020 09:16:08 +0100
Subject: [PATCH 218/716] Cover Scrapy 2.4.1 in the release notes (#4884)

Co-authored-by: Mikhail Korobov <kmike84@gmail.com>
---
 docs/news.rst | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index a3889705d..e92493252 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,29 @@
 Release notes
 =============
 
+.. _release-2.4.1:
+
+Scrapy 2.4.1 (2020-11-17)
+-------------------------
+
+-   Fixed :ref:`feed exports <topics-feed-exports>` overwrite support (:issue:`4845`, :issue:`4857`, :issue:`4859`)
+
+-   Fixed the AsyncIO event loop handling, which could make code hang
+    (:issue:`4855`, :issue:`4872`)
+
+-   Fixed the IPv6-capable DNS resolver
+    :class:`~scrapy.resolver.CachingHostnameResolver` for download handlers
+    that call
+    :meth:`reactor.resolve <twisted.internet.interfaces.IReactorCore.resolve>`
+    (:issue:`4802`, :issue:`4803`)
+
+-   Fixed the output of the :command:`genspider` command showing placeholders
+    instead of the import part of the generated spider module (:issue:`4874`)
+
+-   Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`,
+    :issue:`4876`)
+
+
 .. _release-2.4.0:
 
 Scrapy 2.4.0 (2020-10-11)

From 26836c4e1ae9588ee173c5977fc6611364ca7cc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 17 Nov 2020 09:17:39 +0100
Subject: [PATCH 219/716] =?UTF-8?q?Bump=20version:=202.4.0=20=E2=86=92=202?=
 =?UTF-8?q?.4.1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 0f142472e..956c512cb 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.4.0
+current_version = 2.4.1
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 197c4d5c2..005119baa 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.4.0
+2.4.1

From 63becd1bc89395750b39139e2114193607f3ca61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 17 Nov 2020 21:58:08 +0100
Subject: [PATCH 220/716] Update news.rst

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index e92493252..0391506c4 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -20,7 +20,7 @@ Scrapy 2.4.1 (2020-11-17)
     (:issue:`4802`, :issue:`4803`)
 
 -   Fixed the output of the :command:`genspider` command showing placeholders
-    instead of the import part of the generated spider module (:issue:`4874`)
+    instead of the import path of the generated spider module (:issue:`4874`)
 
 -   Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`,
     :issue:`4876`)

From 6ef3dc2029fea1f692aa3054ea47d799772151d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 17 Nov 2020 22:28:20 +0100
Subject: [PATCH 221/716] Use the new pip resolver for Tox environments with
 pinned dependencies

---
 tox.ini | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tox.ini b/tox.ini
index f35466d05..56e736fbf 100644
--- a/tox.ini
+++ b/tox.ini
@@ -88,12 +88,18 @@ deps =
     botocore==1.4.87
     google-cloud-storage==1.29.0
     Pillow==4.0.0
+install_command =
+    # --use-feature=2020-resolver is required, otherwise the latest verion of
+    # Twisted gets installed.
+    pip install --use-feature=2020-resolver {opts} {packages}
 
 [testenv:pinned]
 deps =
     {[pinned]deps}
     lxml==3.5.0
     PyDispatcher==2.0.5
+install_command =
+    {[pinned]install_command}
     
 [testenv:windows-pinned]
 basepython = python3
@@ -103,6 +109,8 @@ deps =
     # not need to build lxml from sources in a CI Windows job:
     lxml==3.8.0
     PyDispatcher==2.0.5
+install_command =
+    {[pinned]install_command}
 
 [testenv:extra-deps]
 deps =
@@ -110,6 +118,8 @@ deps =
     reppy
     robotexclusionrulesparser
 install_command =
+    # Test --use-feature=2020-resolver for the latest version of all
+    # dependencies.
     pip install --use-feature=2020-resolver {opts} {packages}
 
 [testenv:asyncio]
@@ -118,6 +128,8 @@ commands =
 
 [testenv:asyncio-pinned]
 deps = {[testenv:pinned]deps}
+install_command =
+    {[pinned]install_command}
 commands = {[testenv:asyncio]commands}
 
 [testenv:pypy3]
@@ -131,6 +143,8 @@ deps =
     {[pinned]deps}
     lxml==4.0.0
     PyPyDispatcher==2.1.0
+install_command =
+    {[pinned]install_command}
 commands = {[testenv:pypy3]commands}
 
 [docs]

From bde96a5ad98caed74cbaaeaf1dfe4b215093c03f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 18 Nov 2020 16:42:44 +0100
Subject: [PATCH 222/716] Ignore server-initiated events

---
 scrapy/core/http2/protocol.py | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 9d499596c..67a86bd62 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -349,10 +349,16 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         ])
 
     def data_received(self, event: DataReceived) -> None:
-        self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length)
+        try:
+            self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length)
+        except KeyError:
+            logger.debug(f'Ignoring server-initiated event {event}')
 
     def response_received(self, event: ResponseReceived) -> None:
-        self.streams[event.stream_id].receive_headers(event.headers)
+        try:
+            self.streams[event.stream_id].receive_headers(event.headers)
+        except KeyError:
+            logger.debug(f'Ignoring server-initiated event {event}')
 
     def settings_acknowledged(self, event: SettingsAcknowledged) -> None:
         self.metadata['settings_acknowledged'] = True
@@ -365,12 +371,20 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate())
 
     def stream_ended(self, event: StreamEnded) -> None:
-        stream = self.pop_stream(event.stream_id)
-        stream.close(StreamCloseReason.ENDED, from_protocol=True)
+        try:
+            stream = self.pop_stream(event.stream_id)
+        except KeyError:
+            logger.debug(f'Ignoring server-initiated event {event}')
+        else:
+            stream.close(StreamCloseReason.ENDED, from_protocol=True)
 
     def stream_reset(self, event: StreamReset) -> None:
-        stream = self.pop_stream(event.stream_id)
-        stream.close(StreamCloseReason.RESET, from_protocol=True)
+        try:
+            stream = self.pop_stream(event.stream_id)
+        except KeyError:
+            logger.debug(f'Ignoring server-initiated event {event}')
+        else:
+            stream.close(StreamCloseReason.RESET, from_protocol=True)
 
     def window_updated(self, event: WindowUpdated) -> None:
         if event.stream_id != 0:

From 08f5ed712f4fdafec122f887c3ef06629f35ee0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 18 Nov 2020 17:38:18 +0100
Subject: [PATCH 223/716] Fix memory issue due to unexpectedly large server
 frames

---
 scrapy/core/http2/protocol.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 67a86bd62..d8d0974b8 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -13,7 +13,7 @@ from h2.events import (
     SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived,
     WindowUpdated
 )
-from h2.exceptions import H2Error
+from h2.exceptions import FrameTooLargeError, H2Error
 from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
 from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
@@ -261,6 +261,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             events = self.conn.receive_data(data)
             self._handle_events(events)
         except H2Error as e:
+            if isinstance(e, FrameTooLargeError):
+                # hyper-h2 does not drop the connection in this scenario, we
+                # need to abort the connection manually.
+                self._conn_lost_errors += [e]
+                self.transport.abortConnection()
+                return
+
             # Save this error as ultimately the connection will be dropped
             # internally by hyper-h2. Saved error will be passed to all the streams
             # closed with the connection.

From 075ab156b0ac7bf56828fd40a843f3e9b63a3de3 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Nov 2020 11:59:59 -0300
Subject: [PATCH 224/716] Deprecate scrapy.utils.py36 module

---
 scrapy/utils/py36.py   | 17 +++++++++--------
 scrapy/utils/python.py |  7 +++++++
 scrapy/utils/spider.py |  7 ++-----
 setup.cfg              |  3 ---
 4 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py
index c8c24076e..070b145a4 100644
--- a/scrapy/utils/py36.py
+++ b/scrapy/utils/py36.py
@@ -1,10 +1,11 @@
-"""
-Helpers using Python 3.6+ syntax (ignore SyntaxError on import).
-"""
+import warnings
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.python import collect_asyncgen  # noqa: F401
 
 
-async def collect_asyncgen(result):
-    results = []
-    async for x in result:
-        results.append(x)
-    return results
+warnings.warn(
+    "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.python` instead.",
+    category=ScrapyDeprecationWarning,
+    stacklevel=2,
+)
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 5703fd4c3..7bd286523 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -355,3 +355,10 @@ class MutableChain:
     @deprecated("scrapy.utils.python.MutableChain.__next__")
     def next(self):
         return self.__next__()
+
+
+async def collect_asyncgen(result):
+    results = []
+    async for x in result:
+        results.append(x)
+    return results
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index f3a9a67a3..5319604ba 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -4,17 +4,14 @@ import logging
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
-try:
-    from scrapy.utils.py36 import collect_asyncgen
-except SyntaxError:
-    collect_asyncgen = None
+from scrapy.utils.python import collect_asyncgen
 
 
 logger = logging.getLogger(__name__)
 
 
 def iterate_spider_output(result):
-    if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result):
+    if inspect.isasyncgen(result):
         d = deferred_from_coro(collect_asyncgen(result))
         d.addCallback(iterate_spider_output)
         return d
diff --git a/setup.cfg b/setup.cfg
index 8101443e3..0c9f6b963 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -55,9 +55,6 @@ ignore_errors = True
 [mypy-scrapy.utils.response]
 ignore_errors = True
 
-[mypy-scrapy.utils.spider]
-ignore_errors = True
-
 [mypy-scrapy.utils.trackref]
 ignore_errors = True
 

From 4075e1eadd0a9e83356d18b9ca73090e4a0bc770 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Nov 2020 15:07:56 -0300
Subject: [PATCH 225/716] Remove deprecated modules
 (utils.http/markup/multipart)

---
 pytest.ini                |  3 ---
 scrapy/utils/http.py      | 36 ------------------------------------
 scrapy/utils/markup.py    | 14 --------------
 scrapy/utils/multipart.py | 15 ---------------
 4 files changed, 68 deletions(-)
 delete mode 100644 scrapy/utils/http.py
 delete mode 100644 scrapy/utils/markup.py
 delete mode 100644 scrapy/utils/multipart.py

diff --git a/pytest.ini b/pytest.ini
index 1c95f715a..d4deeb57c 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -36,8 +36,5 @@ flake8-ignore =
     scrapy/spiders/__init__.py E402 F401
 
     # Issues pending a review:
-    scrapy/utils/http.py F403
-    scrapy/utils/markup.py F403
-    scrapy/utils/multipart.py F403
     scrapy/utils/url.py F403 F405
     tests/test_loader.py E741
diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py
deleted file mode 100644
index ceb3f0509..000000000
--- a/scrapy/utils/http.py
+++ /dev/null
@@ -1,36 +0,0 @@
-"""
-Transitional module for moving to the w3lib library.
-
-For new code, always import from w3lib.http instead of this module
-"""
-
-import warnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.decorators import deprecated
-from w3lib.http import *  # noqa: F401
-
-
-warnings.warn("Module `scrapy.utils.http` is deprecated, "
-              "Please import from `w3lib.http` instead.",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-
-@deprecated
-def decode_chunked_transfer(chunked_body):
-    """Parsed body received with chunked transfer encoding, and return the
-    decoded body.
-
-    For more info see:
-    https://en.wikipedia.org/wiki/Chunked_transfer_encoding
-
-    """
-    body, h, t = '', '', chunked_body
-    while t:
-        h, t = t.split('\r\n', 1)
-        if h == '0':
-            break
-        size = int(h, 16)
-        body += t[:size]
-        t = t[size + 2:]
-    return body
diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py
deleted file mode 100644
index 9728c542a..000000000
--- a/scrapy/utils/markup.py
+++ /dev/null
@@ -1,14 +0,0 @@
-"""
-Transitional module for moving to the w3lib library.
-
-For new code, always import from w3lib.html instead of this module
-"""
-import warnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from w3lib.html import *  # noqa: F401
-
-
-warnings.warn("Module `scrapy.utils.markup` is deprecated. "
-              "Please import from `w3lib.html` instead.",
-              ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py
deleted file mode 100644
index 5dcf791b8..000000000
--- a/scrapy/utils/multipart.py
+++ /dev/null
@@ -1,15 +0,0 @@
-"""
-Transitional module for moving to the w3lib library.
-
-For new code, always import from w3lib.form instead of this module
-"""
-import warnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from w3lib.form import *  # noqa: F401
-
-
-warnings.warn("Module `scrapy.utils.multipart` is deprecated. "
-              "If you're using `encode_multipart` function, please use "
-              "`urllib3.filepost.encode_multipart_formdata` instead",
-              ScrapyDeprecationWarning, stacklevel=2)

From 51ca4d0138e5c9cf637074f59c839ff9b5839db6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Nov 2020 15:47:08 -0300
Subject: [PATCH 226/716] Remove deprecated scrapy.utils.gz.is_gzipped function

---
 scrapy/utils/gz.py     | 15 +-----------
 tests/test_utils_gz.py | 55 ++++++++++++------------------------------
 2 files changed, 16 insertions(+), 54 deletions(-)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 11d433cf5..76156a4b8 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -1,7 +1,6 @@
+import struct
 from gzip import GzipFile
 from io import BytesIO
-import re
-import struct
 
 from scrapy.utils.decorators import deprecated
 
@@ -42,17 +41,5 @@ def gunzip(data):
     return b''.join(output_list)
 
 
-_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
-_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
-
-
-@deprecated
-def is_gzipped(response):
-    """Return True if the response is gzipped, or False otherwise"""
-    ctype = response.headers.get('Content-Type', b'')
-    cenc = response.headers.get('Content-Encoding', b'').lower()
-    return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
-
-
 def gzip_magic_number(response):
     return response.body[:3] == b'\x1f\x8b\x08'
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index 7148185f4..4943731cb 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -3,10 +3,11 @@ from os.path import join
 
 from w3lib.encoding import html_to_unicode
 
-from scrapy.utils.gz import gunzip, is_gzipped
-from scrapy.http import Response, Headers
+from scrapy.utils.gz import gunzip, gzip_magic_number
+from scrapy.http import Response
 from tests import tests_datadir
 
+
 SAMPLEDIR = join(tests_datadir, 'compressed')
 
 
@@ -14,8 +15,12 @@ class GunzipTest(unittest.TestCase):
 
     def test_gunzip_basic(self):
         with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f:
-            text = gunzip(f.read())
-            self.assertEqual(len(text), 9950)
+            r1 = Response("http://www.example.com", body=f.read())
+            self.assertTrue(gzip_magic_number(r1))
+
+            r2 = Response("http://www.example.com", body=gunzip(r1.body))
+            self.assertFalse(gzip_magic_number(r2))
+            self.assertEqual(len(r2.body), 9950)
 
     def test_gunzip_truncated(self):
         with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f:
@@ -28,46 +33,16 @@ class GunzipTest(unittest.TestCase):
 
     def test_gunzip_truncated_short(self):
         with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f:
-            text = gunzip(f.read())
-            assert text.endswith(b'</html>')
+            r1 = Response("http://www.example.com", body=f.read())
+            self.assertTrue(gzip_magic_number(r1))
 
-    def test_is_x_gzipped_right(self):
-        hdrs = Headers({"Content-Type": "application/x-gzip"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertTrue(is_gzipped(r1))
-
-    def test_is_gzipped_right(self):
-        hdrs = Headers({"Content-Type": "application/gzip"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertTrue(is_gzipped(r1))
-
-    def test_is_gzipped_not_quite(self):
-        hdrs = Headers({"Content-Type": "application/gzippppp"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertFalse(is_gzipped(r1))
-
-    def test_is_gzipped_case_insensitive(self):
-        hdrs = Headers({"Content-Type": "Application/X-Gzip"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertTrue(is_gzipped(r1))
-
-        hdrs = Headers({"Content-Type": "application/X-GZIP ; charset=utf-8"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertTrue(is_gzipped(r1))
+            r2 = Response("http://www.example.com", body=gunzip(r1.body))
+            assert r2.body.endswith(b'</html>')
+            self.assertFalse(gzip_magic_number(r2))
 
     def test_is_gzipped_empty(self):
         r1 = Response("http://www.example.com")
-        self.assertFalse(is_gzipped(r1))
-
-    def test_is_gzipped_wrong(self):
-        hdrs = Headers({"Content-Type": "application/javascript"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertFalse(is_gzipped(r1))
-
-    def test_is_gzipped_with_charset(self):
-        hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"})
-        r1 = Response("http://www.example.com", headers=hdrs)
-        self.assertTrue(is_gzipped(r1))
+        self.assertFalse(gzip_magic_number(r1))
 
     def test_gunzip_illegal_eof(self):
         with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f:

From 462014bc5738a5ed18ec4c15a91384eb17e57096 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Nov 2020 15:51:59 -0300
Subject: [PATCH 227/716] Scheduler: remove support for deprecated
 queuelib.PriorityQueue

---
 scrapy/core/scheduler.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index a18c26b17..9ce823dbc 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -1,14 +1,10 @@
 import os
 import json
 import logging
-import warnings
 from os.path import join, exists
 
-from queuelib import PriorityQueue
-
 from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.job import job_dir
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
 
 
 logger = logging.getLogger(__name__)
@@ -56,14 +52,6 @@ class Scheduler:
         dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
         dupefilter = create_instance(dupefilter_cls, settings, crawler)
         pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
-        if pqclass is PriorityQueue:
-            warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
-                          " is no longer supported because of API changes; "
-                          "please use 'scrapy.pqueues.ScrapyPriorityQueue'",
-                          ScrapyDeprecationWarning)
-            from scrapy.pqueues import ScrapyPriorityQueue
-            pqclass = ScrapyPriorityQueue
-
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
         logunser = settings.getbool('SCHEDULER_DEBUG')

From 0a93df9efd20a4bcd34d2ee5e6bdf6d365d70409 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Nov 2020 16:16:18 -0300
Subject: [PATCH 228/716] Move collect_asyncgen to utils.asyncgen

---
 scrapy/utils/asyncgen.py | 5 +++++
 scrapy/utils/py36.py     | 4 ++--
 scrapy/utils/python.py   | 7 -------
 3 files changed, 7 insertions(+), 9 deletions(-)
 create mode 100644 scrapy/utils/asyncgen.py

diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
new file mode 100644
index 000000000..7f697af5f
--- /dev/null
+++ b/scrapy/utils/asyncgen.py
@@ -0,0 +1,5 @@
+async def collect_asyncgen(result):
+    results = []
+    async for x in result:
+        results.append(x)
+    return results
diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py
index 070b145a4..653e2bbbb 100644
--- a/scrapy/utils/py36.py
+++ b/scrapy/utils/py36.py
@@ -1,11 +1,11 @@
 import warnings
 
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.python import collect_asyncgen  # noqa: F401
+from scrapy.utils.asyncgen import collect_asyncgen  # noqa: F401
 
 
 warnings.warn(
-    "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.python` instead.",
+    "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.",
     category=ScrapyDeprecationWarning,
     stacklevel=2,
 )
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 7bd286523..5703fd4c3 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -355,10 +355,3 @@ class MutableChain:
     @deprecated("scrapy.utils.python.MutableChain.__next__")
     def next(self):
         return self.__next__()
-
-
-async def collect_asyncgen(result):
-    results = []
-    async for x in result:
-        results.append(x)
-    return results

From 18b05af87783d71f9c8f9f1ebe027083037e6f86 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Nov 2020 16:18:58 -0300
Subject: [PATCH 229/716] Remove tests/test_utils_http.py

---
 tests/test_utils_http.py | 19 -------------------
 1 file changed, 19 deletions(-)
 delete mode 100644 tests/test_utils_http.py

diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py
deleted file mode 100644
index 363b015a8..000000000
--- a/tests/test_utils_http.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import unittest
-
-from scrapy.utils.http import decode_chunked_transfer
-
-
-class ChunkedTest(unittest.TestCase):
-
-    def test_decode_chunked_transfer(self):
-        """Example taken from: http://en.wikipedia.org/wiki/Chunked_transfer_encoding"""
-        chunked_body = "25\r\n" + "This is the data in the first chunk\r\n\r\n"
-        chunked_body += "1C\r\n" + "and this is the second one\r\n\r\n"
-        chunked_body += "3\r\n" + "con\r\n"
-        chunked_body += "8\r\n" + "sequence\r\n"
-        chunked_body += "0\r\n\r\n"
-        body = decode_chunked_transfer(chunked_body)
-        self.assertEqual(
-            body,
-            "This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
-        )

From fe8bb6bd905ad2a733ba5e876f4197888605902e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 23 Nov 2020 16:51:04 -0300
Subject: [PATCH 230/716] Fix import

---
 scrapy/utils/spider.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 5319604ba..59fc9202f 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -4,7 +4,7 @@ import logging
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.python import collect_asyncgen
+from scrapy.utils.asyncgen import collect_asyncgen
 
 
 logger = logging.getLogger(__name__)

From 0dc3e6350c230028addd8e73833c93341bab4b42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 23 Nov 2020 22:10:45 +0100
Subject: [PATCH 231/716] Add a test to check the Twisted version in pinned
 environments

---
 tests/test_dependencies.py | 32 ++++++++++++++++++++++++++++++++
 tox.ini                    | 10 ++++++++++
 2 files changed, 42 insertions(+)

diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index 5d0a1d0c9..4e4f190ab 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -1,8 +1,14 @@
+import os
+import re
+from configparser import ConfigParser
 from importlib import import_module
+
+from twisted import version as twisted_version
 from twisted.trial import unittest
 
 
 class ScrapyUtilsTest(unittest.TestCase):
+
     def test_required_openssl_version(self):
         try:
             module = import_module('OpenSSL')
@@ -13,6 +19,32 @@ class ScrapyUtilsTest(unittest.TestCase):
             installed_version = [int(x) for x in module.__version__.split('.')[:2]]
             assert installed_version >= [0, 6], "OpenSSL >= 0.6 required"
 
+    def test_pinned_twisted_version(self):
+        """When running tests within a Tox environment with pinned
+        dependencies, make sure that the version of Twisted is the pinned
+        version.
+
+        See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011
+        """
+        if not os.environ.get('SCRAPY_PINNED', None):
+            self.skipTest('Not in a pinned environment')
+
+        tox_config_file_path = os.path.join(
+            os.path.dirname(__file__),
+            '..',
+            'tox.ini',
+        )
+        config_parser = ConfigParser()
+        config_parser.read(tox_config_file_path)
+        pattern = r'Twisted==([\d.]+)'
+        match = re.search(pattern, config_parser['pinned']['deps'])
+        pinned_twisted_version_string = match[1]
+
+        self.assertEqual(
+            twisted_version.short(),
+            pinned_twisted_version_string
+        )
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tox.ini b/tox.ini
index 56e736fbf..ea71a2476 100644
--- a/tox.ini
+++ b/tox.ini
@@ -92,6 +92,8 @@ install_command =
     # --use-feature=2020-resolver is required, otherwise the latest verion of
     # Twisted gets installed.
     pip install --use-feature=2020-resolver {opts} {packages}
+setenv =
+    SCRAPY_PINNED=true
 
 [testenv:pinned]
 deps =
@@ -100,6 +102,8 @@ deps =
     PyDispatcher==2.0.5
 install_command =
     {[pinned]install_command}
+setenv =
+    {[pinned]setenv}
     
 [testenv:windows-pinned]
 basepython = python3
@@ -111,6 +115,8 @@ deps =
     PyDispatcher==2.0.5
 install_command =
     {[pinned]install_command}
+setenv =
+    {[pinned]setenv}
 
 [testenv:extra-deps]
 deps =
@@ -131,6 +137,8 @@ deps = {[testenv:pinned]deps}
 install_command =
     {[pinned]install_command}
 commands = {[testenv:asyncio]commands}
+setenv =
+    {[pinned]setenv}
 
 [testenv:pypy3]
 basepython = pypy3
@@ -146,6 +154,8 @@ deps =
 install_command =
     {[pinned]install_command}
 commands = {[testenv:pypy3]commands}
+setenv =
+    {[pinned]setenv}
 
 [docs]
 changedir = docs

From a752fa072e1acbb233191dbf04728db1fd6712a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 23 Nov 2020 22:58:54 +0100
Subject: [PATCH 232/716] Implement retry request functions and mixin

---
 scrapy/downloadermiddlewares/retry.py | 134 ++++++++++++++++++++------
 1 file changed, 106 insertions(+), 28 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 51fe59254..023ab7d60 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -31,6 +31,105 @@ from scrapy.utils.python import global_object_name
 logger = logging.getLogger(__name__)
 
 
+def get_retry_request(
+    request,
+    *,
+    reason,
+    spider,
+    max_retry_times=None,
+    priority_adjust=None,
+):
+    settings = spider.crawler.settings
+    stats = spider.crawler.stats
+    retry_times = request.meta.get('retry_times', 0) + 1
+    request_max_retry_times = request.meta.get(
+        'max_retry_times',
+        max_retry_times,
+    )
+    if request_max_retry_times is None:
+        request_max_retry_times = settings.getint('RETRY_TIMES')
+    if retry_times <= request_max_retry_times:
+        logger.debug(
+            "Retrying %(request)s (failed %(retry_times)d times): %(reason)s",
+            {'request': request, 'retry_times': retry_times, 'reason': reason},
+            extra={'spider': spider}
+        )
+        new_request = request.copy()
+        new_request.meta['retry_times'] = retry_times
+        new_request.dont_filter = True
+        if priority_adjust is None:
+            priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
+        new_request.priority = request.priority + priority_adjust
+
+        if isinstance(reason, Exception):
+            reason = global_object_name(reason.__class__)
+
+        stats.inc_value('retry/count')
+        stats.inc_value(f'retry/reason_count/{reason}')
+        return new_request
+    else:
+        stats.inc_value('retry/max_reached')
+        logger.error("Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s",
+                        {'request': request, 'retry_times': retry_times, 'reason': reason},
+                        extra={'spider': spider})
+        return None
+
+
+def retry_request(
+    request,
+    *,
+    reason,
+    spider,
+    max_retry_times=None,
+    priority_adjust=None,
+):
+    new_request = get_retry_request(
+        request,
+        reason=reason,
+        spider=spider,
+        max_retry_times=max_retry_times,
+        priority_adjust=priority_adjust,
+    )
+    if new_request:
+        return [new_request]
+    return []
+
+
+class RetrySpiderMixin:
+
+    def get_retry_request(
+        self,
+        request,
+        *,
+        reason,
+        max_retry_times=None,
+        priority_adjust=None,
+    ):
+        return get_retry_request(
+            request,
+            reason=reason,
+            spider=self,
+            max_retry_times=max_retry_times,
+            priority_adjust=priority_adjust,
+        )
+
+    def retry_request(
+        self,
+        request,
+        *,
+        reason,
+        max_retry_times=None,
+        priority_adjust=None,
+    ):
+        return retry_request(
+            request,
+            reason=reason,
+            spider=self,
+            max_retry_times=max_retry_times,
+            priority_adjust=priority_adjust,
+        )
+
+
 class RetryMiddleware:
 
     # IOError is raised by the HttpCompression middleware when trying to
@@ -67,31 +166,10 @@ class RetryMiddleware:
             return self._retry(request, exception, spider)
 
     def _retry(self, request, reason, spider):
-        retries = request.meta.get('retry_times', 0) + 1
-
-        retry_times = self.max_retry_times
-
-        if 'max_retry_times' in request.meta:
-            retry_times = request.meta['max_retry_times']
-
-        stats = spider.crawler.stats
-        if retries <= retry_times:
-            logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
-                         {'request': request, 'retries': retries, 'reason': reason},
-                         extra={'spider': spider})
-            retryreq = request.copy()
-            retryreq.meta['retry_times'] = retries
-            retryreq.dont_filter = True
-            retryreq.priority = request.priority + self.priority_adjust
-
-            if isinstance(reason, Exception):
-                reason = global_object_name(reason.__class__)
-
-            stats.inc_value('retry/count')
-            stats.inc_value(f'retry/reason_count/{reason}')
-            return retryreq
-        else:
-            stats.inc_value('retry/max_reached')
-            logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
-                         {'request': request, 'retries': retries, 'reason': reason},
-                         extra={'spider': spider})
+        return get_retry_request(
+            request,
+            reason=reason,
+            spider=spider,
+            max_retry_times=self.max_retry_times,
+            priority_adjust=self.priority_adjust,
+        )

From f6879c681ebf94b22ee777f3249dfe59a99559cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 23 Nov 2020 23:53:03 +0100
Subject: [PATCH 233/716] =?UTF-8?q?SCRAPY=5FPINNED=20=E2=86=92=20=5FSCRAPY?=
 =?UTF-8?q?=5FPINNED?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tests/test_dependencies.py | 2 +-
 tox.ini                    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index 4e4f190ab..93e7311d2 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -26,7 +26,7 @@ class ScrapyUtilsTest(unittest.TestCase):
 
         See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011
         """
-        if not os.environ.get('SCRAPY_PINNED', None):
+        if not os.environ.get('_SCRAPY_PINNED', None):
             self.skipTest('Not in a pinned environment')
 
         tox_config_file_path = os.path.join(
diff --git a/tox.ini b/tox.ini
index ea71a2476..66866301c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -93,7 +93,7 @@ install_command =
     # Twisted gets installed.
     pip install --use-feature=2020-resolver {opts} {packages}
 setenv =
-    SCRAPY_PINNED=true
+    _SCRAPY_PINNED=true
 
 [testenv:pinned]
 deps =

From 95d39d5cb464ca22516a30f96c0a323613421090 Mon Sep 17 00:00:00 2001
From: etimoz <etimoz@users.noreply.github.com>
Date: Sun, 29 Nov 2020 13:24:04 +0100
Subject: [PATCH 234/716] removed wrong super argument in overriding
 serialize_fields code example

---
 docs/topics/exporters.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index ef50c9f5c..0a0a1765a 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -123,7 +123,7 @@ Example::
           def serialize_field(self, field, name, value):
               if field == 'price':
                   return f'$ {str(value)}'
-              return super(Product, self).serialize_field(field, name, value)
+              return super().serialize_field(field, name, value)
 
 .. _topics-exporters-reference:
 

From 7fec9f991f0bd415900df2bf288c6cda909ecd77 Mon Sep 17 00:00:00 2001
From: Kader DJEHAF <daek@online.fr>
Date: Mon, 30 Nov 2020 21:47:28 +0100
Subject: [PATCH 235/716] [Cleaned] PEP 8: E251 unexpected spaces around
 keyword / parameter equals (#4911)

[Cleaned] PEP 8: E251 unexpected spaces around keyword / parameter equals
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 0c2281400..35736b75f 100644
--- a/setup.py
+++ b/setup.py
@@ -56,7 +56,7 @@ setup(
     name='Scrapy',
     version=version,
     url='https://scrapy.org',
-    project_urls = {
+    project_urls={
         'Documentation': 'https://docs.scrapy.org/',
         'Source': 'https://github.com/scrapy/scrapy',
         'Tracker': 'https://github.com/scrapy/scrapy/issues',

From a80bafe5cdcb4b5e2542524fe641570f9107a121 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 30 Nov 2020 19:03:13 -0300
Subject: [PATCH 236/716] Remove deprecated SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE

---
 scrapy/utils/project.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index fd13d85e3..35e59a258 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -68,18 +68,10 @@ def get_project_settings():
     if settings_module_path:
         settings.setmodule(settings_module_path, priority='project')
 
-    pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
-    if pickled_settings:
-        warnings.warn("Use of environment variable "
-                      "'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE' "
-                      "is deprecated.", ScrapyDeprecationWarning)
-        settings.setdict(pickle.loads(pickled_settings), priority='project')
-
     scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if
                       k.startswith('SCRAPY_')}
     valid_envvars = {
         'CHECK',
-        'PICKLED_SETTINGS_TO_OVERRIDE',
         'PROJECT',
         'PYTHON_SHELL',
         'SETTINGS_MODULE',

From 6091f3cc03835b24def08bf358d7b19207be685d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 1 Dec 2020 10:26:21 -0300
Subject: [PATCH 237/716] Remove unused pickle import

---
 scrapy/utils/project.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index 35e59a258..c66af497e 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -1,5 +1,4 @@
 import os
-import pickle
 import warnings
 
 from importlib import import_module

From ef09e0d10fc950ac308a159988be6b50e87bd906 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 19 Nov 2020 10:35:49 -0300
Subject: [PATCH 238/716] Some type hints

---
 scrapy/__init__.py                  |   2 +-
 scrapy/commands/__init__.py         |   4 +-
 scrapy/commands/parse.py            |   6 +-
 scrapy/contracts/__init__.py        | 127 ++++++++++++++--------------
 scrapy/http/cookies.py              |   8 +-
 scrapy/item.py                      |   3 +-
 scrapy/mail.py                      |   7 +-
 scrapy/spidermiddlewares/referer.py |  30 ++++---
 scrapy/utils/httpobj.py             |  11 ++-
 scrapy/utils/request.py             |  28 +++---
 scrapy/utils/response.py            |  31 ++++---
 scrapy/utils/trackref.py            |   9 +-
 setup.cfg                           |  36 --------
 13 files changed, 154 insertions(+), 148 deletions(-)

diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 4326ca4aa..8a8065bf2 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -22,7 +22,7 @@ __all__ = [
 
 
 # Scrapy and Twisted versions
-__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
+__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip()
 version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
 twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 23ccffcd9..6e77551c6 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -3,6 +3,8 @@ Base class for Scrapy commands
 """
 import os
 from optparse import OptionGroup
+from typing import Any, Dict
+
 from twisted.python import failure
 
 from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
@@ -15,7 +17,7 @@ class ScrapyCommand:
     crawler_process = None
 
     # default settings to be used for this command instead of global defaults
-    default_settings = {}
+    default_settings: Dict[str, Any] = {}
 
     exitcode = 0
 
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 83ee074da..52118db1b 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,5 +1,6 @@
 import json
 import logging
+from typing import Dict
 
 from itemadapter import is_item, ItemAdapter
 from w3lib.url import is_url
@@ -10,6 +11,7 @@ from scrapy.utils import display
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
 from scrapy.exceptions import UsageError
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -17,8 +19,8 @@ class Command(BaseRunSpiderCommand):
     requires_project = True
 
     spider = None
-    items = {}
-    requests = {}
+    items: Dict[int, list] = {}
+    requests: Dict[int, list] = {}
 
     first_response = None
 
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index db0a56e56..b47e55092 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -1,16 +1,77 @@
-import sys
 import re
+import sys
 from functools import wraps
 from inspect import getmembers
+from typing import Dict
 from unittest import TestCase
 
 from scrapy.http import Request
-from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.python import get_spec
+from scrapy.utils.spider import iterate_spider_output
+
+
+class Contract:
+    """ Abstract class for contracts """
+    request_cls = None
+
+    def __init__(self, method, *args):
+        self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
+        self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
+        self.args = args
+
+    def add_pre_hook(self, request, results):
+        if hasattr(self, 'pre_process'):
+            cb = request.callback
+
+            @wraps(cb)
+            def wrapper(response, **cb_kwargs):
+                try:
+                    results.startTest(self.testcase_pre)
+                    self.pre_process(response)
+                    results.stopTest(self.testcase_pre)
+                except AssertionError:
+                    results.addFailure(self.testcase_pre, sys.exc_info())
+                except Exception:
+                    results.addError(self.testcase_pre, sys.exc_info())
+                else:
+                    results.addSuccess(self.testcase_pre)
+                finally:
+                    return list(iterate_spider_output(cb(response, **cb_kwargs)))
+
+            request.callback = wrapper
+
+        return request
+
+    def add_post_hook(self, request, results):
+        if hasattr(self, 'post_process'):
+            cb = request.callback
+
+            @wraps(cb)
+            def wrapper(response, **cb_kwargs):
+                output = list(iterate_spider_output(cb(response, **cb_kwargs)))
+                try:
+                    results.startTest(self.testcase_post)
+                    self.post_process(output)
+                    results.stopTest(self.testcase_post)
+                except AssertionError:
+                    results.addFailure(self.testcase_post, sys.exc_info())
+                except Exception:
+                    results.addError(self.testcase_post, sys.exc_info())
+                else:
+                    results.addSuccess(self.testcase_post)
+                finally:
+                    return output
+
+            request.callback = wrapper
+
+        return request
+
+    def adjust_request_args(self, args):
+        return args
 
 
 class ContractsManager:
-    contracts = {}
+    contracts: Dict[str, Contract] = {}
 
     def __init__(self, contracts):
         for contract in contracts:
@@ -107,66 +168,6 @@ class ContractsManager:
         request.errback = eb_wrapper
 
 
-class Contract:
-    """ Abstract class for contracts """
-    request_cls = None
-
-    def __init__(self, method, *args):
-        self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook')
-        self.testcase_post = _create_testcase(method, f'@{self.name} post-hook')
-        self.args = args
-
-    def add_pre_hook(self, request, results):
-        if hasattr(self, 'pre_process'):
-            cb = request.callback
-
-            @wraps(cb)
-            def wrapper(response, **cb_kwargs):
-                try:
-                    results.startTest(self.testcase_pre)
-                    self.pre_process(response)
-                    results.stopTest(self.testcase_pre)
-                except AssertionError:
-                    results.addFailure(self.testcase_pre, sys.exc_info())
-                except Exception:
-                    results.addError(self.testcase_pre, sys.exc_info())
-                else:
-                    results.addSuccess(self.testcase_pre)
-                finally:
-                    return list(iterate_spider_output(cb(response, **cb_kwargs)))
-
-            request.callback = wrapper
-
-        return request
-
-    def add_post_hook(self, request, results):
-        if hasattr(self, 'post_process'):
-            cb = request.callback
-
-            @wraps(cb)
-            def wrapper(response, **cb_kwargs):
-                output = list(iterate_spider_output(cb(response, **cb_kwargs)))
-                try:
-                    results.startTest(self.testcase_post)
-                    self.post_process(output)
-                    results.stopTest(self.testcase_post)
-                except AssertionError:
-                    results.addFailure(self.testcase_post, sys.exc_info())
-                except Exception:
-                    results.addError(self.testcase_post, sys.exc_info())
-                else:
-                    results.addSuccess(self.testcase_post)
-                finally:
-                    return output
-
-            request.callback = wrapper
-
-        return request
-
-    def adjust_request_args(self, args):
-        return args
-
-
 def _create_testcase(method, desc):
     spider = method.__self__.name
 
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 0c97e6999..bf4ae7b45 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -1,10 +1,16 @@
+import re
 import time
-from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy
 
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 
 
+# Defined in the http.cookiejar module, but undocumented:
+# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527
+IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
+
+
 class CookieJar:
     def __init__(self, policy=None, check_expired_frequency=10000):
         self.policy = policy or DefaultCookiePolicy()
diff --git a/scrapy/item.py b/scrapy/item.py
index af3849302..2ccd7ad18 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -8,6 +8,7 @@ from abc import ABCMeta
 from collections.abc import MutableMapping
 from copy import deepcopy
 from pprint import pformat
+from typing import Dict
 from warnings import warn
 
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
@@ -75,7 +76,7 @@ class ItemMeta(_BaseItemMeta):
 
 class DictItem(MutableMapping, BaseItem):
 
-    fields = {}
+    fields: Dict[str, Field] = {}
 
     def __new__(cls, *args, **kwargs):
         if issubclass(cls, DictItem) and not issubclass(cls, Item):
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 7d7a2c435..2a25ccd44 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -9,7 +9,7 @@ from email.mime.base import MIMEBase
 from email.mime.multipart import MIMEMultipart
 from email.mime.nonmultipart import MIMENonMultipart
 from email.mime.text import MIMEText
-from email.utils import COMMASPACE, formatdate
+from email.utils import formatdate
 from io import BytesIO
 
 from twisted.internet import defer, ssl
@@ -21,6 +21,11 @@ from scrapy.utils.python import to_bytes
 logger = logging.getLogger(__name__)
 
 
+# Defined in the email.utils module, but undocumented:
+# https://github.com/python/cpython/blob/v3.9.0/Lib/email/utils.py#L42
+COMMASPACE = ", "
+
+
 def _to_bytes_or_none(text):
     if text is None:
         return None
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index f81041376..608c0eea5 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -3,15 +3,16 @@ RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
 import warnings
+from typing import Tuple
 from urllib.parse import urlparse
 
 from w3lib.url import safe_url_string
 
-from scrapy.http import Request, Response
-from scrapy.exceptions import NotConfigured
 from scrapy import signals
-from scrapy.utils.python import to_unicode
+from scrapy.exceptions import NotConfigured
+from scrapy.http import Request, Response
 from scrapy.utils.misc import load_object
+from scrapy.utils.python import to_unicode
 from scrapy.utils.url import strip_url
 
 
@@ -30,7 +31,8 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default"
 
 class ReferrerPolicy:
 
-    NOREFERRER_SCHEMES = LOCAL_SCHEMES
+    NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES
+    name: str
 
     def referrer(self, response_url, request_url):
         raise NotImplementedError()
@@ -88,7 +90,7 @@ class NoReferrerPolicy(ReferrerPolicy):
     is to be sent along with requests made from a particular request client to any origin.
     The header will be omitted entirely.
     """
-    name = POLICY_NO_REFERRER
+    name: str = POLICY_NO_REFERRER
 
     def referrer(self, response_url, request_url):
         return None
@@ -108,7 +110,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
 
     This is a user agent's default behavior, if no policy is otherwise specified.
     """
-    name = POLICY_NO_REFERRER_WHEN_DOWNGRADE
+    name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE
 
     def referrer(self, response_url, request_url):
         if not self.tls_protected(response_url) or self.tls_protected(request_url):
@@ -125,7 +127,7 @@ class SameOriginPolicy(ReferrerPolicy):
     Cross-origin requests, on the other hand, will contain no referrer information.
     A Referer HTTP header will not be sent.
     """
-    name = POLICY_SAME_ORIGIN
+    name: str = POLICY_SAME_ORIGIN
 
     def referrer(self, response_url, request_url):
         if self.origin(response_url) == self.origin(request_url):
@@ -141,7 +143,7 @@ class OriginPolicy(ReferrerPolicy):
     when making both same-origin requests and cross-origin requests
     from a particular request client.
     """
-    name = POLICY_ORIGIN
+    name: str = POLICY_ORIGIN
 
     def referrer(self, response_url, request_url):
         return self.origin_referrer(response_url)
@@ -160,7 +162,7 @@ class StrictOriginPolicy(ReferrerPolicy):
     on the other hand, will contain no referrer information.
     A Referer HTTP header will not be sent.
     """
-    name = POLICY_STRICT_ORIGIN
+    name: str = POLICY_STRICT_ORIGIN
 
     def referrer(self, response_url, request_url):
         if (
@@ -181,7 +183,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
     is sent as referrer information when making cross-origin requests
     from a particular request client.
     """
-    name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
+    name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
 
     def referrer(self, response_url, request_url):
         origin = self.origin(response_url)
@@ -208,7 +210,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
     on the other hand, will contain no referrer information.
     A Referer HTTP header will not be sent.
     """
-    name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
+    name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
 
     def referrer(self, response_url, request_url):
         origin = self.origin(response_url)
@@ -234,7 +236,7 @@ class UnsafeUrlPolicy(ReferrerPolicy):
     to insecure origins.
     Carefully consider the impact of setting such a policy for potentially sensitive documents.
     """
-    name = POLICY_UNSAFE_URL
+    name: str = POLICY_UNSAFE_URL
 
     def referrer(self, response_url, request_url):
         return self.stripped_referrer(response_url)
@@ -246,8 +248,8 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
     with the addition that "Referer" is not sent if the parent request was
     using ``file://`` or ``s3://`` scheme.
     """
-    NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
-    name = POLICY_SCRAPY_DEFAULT
+    NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ('file', 's3')
+    name: str = POLICY_SCRAPY_DEFAULT
 
 
 _policy_classes = {p.name: p for p in (
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index c8d4391b1..a90f1d278 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -1,13 +1,16 @@
 """Helper functions for scrapy.http objects (Request, Response)"""
 
-import weakref
-from urllib.parse import urlparse
+from typing import Union
+from urllib.parse import urlparse, ParseResult
+from weakref import WeakKeyDictionary
+
+from scrapy.http import Request, Response
 
 
-_urlparse_cache = weakref.WeakKeyDictionary()
+_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary()
 
 
-def urlparse_cached(request_or_response):
+def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult:
     """Return urlparse.urlparse caching the result, where the argument can be a
     Request or Response object
     """
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 12c03d78e..66736b42f 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -4,20 +4,27 @@ scrapy.http.Request objects
 """
 
 import hashlib
-import weakref
+from typing import Dict, Iterable, Optional, Tuple, Union
 from urllib.parse import urlunparse
+from weakref import WeakKeyDictionary
 
 from w3lib.http import basic_auth_header
 from w3lib.url import canonicalize_url
 
+from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes, to_unicode
 
 
-_fingerprint_cache = weakref.WeakKeyDictionary()
+_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
+_fingerprint_cache = WeakKeyDictionary()
 
 
-def request_fingerprint(request, include_headers=None, keep_fragments=False):
+def request_fingerprint(
+    request: Request,
+    include_headers: Optional[Iterable[Union[bytes, str]]] = None,
+    keep_fragments: bool = False,
+):
     """
     Return the request fingerprint.
 
@@ -49,17 +56,18 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
     (for instance when handling requests with a headless browser).
 
     """
+    headers: Optional[Tuple[bytes, ...]] = None
     if include_headers:
-        include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
+        headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
     cache = _fingerprint_cache.setdefault(request, {})
-    cache_key = (include_headers, keep_fragments)
+    cache_key = (headers, keep_fragments)
     if cache_key not in cache:
         fp = hashlib.sha1()
         fp.update(to_bytes(request.method))
         fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
         fp.update(request.body or b'')
-        if include_headers:
-            for hdr in include_headers:
+        if headers:
+            for hdr in headers:
                 if hdr in request.headers:
                     fp.update(hdr)
                     for v in request.headers.getlist(hdr):
@@ -68,14 +76,14 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
     return cache[cache_key]
 
 
-def request_authenticate(request, username, password):
+def request_authenticate(request: Request, username: str, password: str) -> None:
     """Autenticate the given request (in place) using the HTTP basic access
     authentication mechanism (RFC 2617) and the given username and password
     """
     request.headers['Authorization'] = basic_auth_header(username, password)
 
 
-def request_httprepr(request):
+def request_httprepr(request: Request) -> bytes:
     """Return the raw HTTP representation (as bytes) of the given request.
     This is provided only for reference since it's not the actual stream of
     bytes that will be send when performing the request (that's controlled
@@ -92,7 +100,7 @@ def request_httprepr(request):
     return s
 
 
-def referer_str(request):
+def referer_str(request: Request) -> Optional[str]:
     """ Return Referer HTTP header suitable for logging. """
     referrer = request.headers.get('Referer')
     if referrer is None:
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 99b089b6f..b3ef7b463 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -3,19 +3,23 @@ This module provides some useful functions for working with
 scrapy.http.Response objects
 """
 import os
-import weakref
 import webbrowser
 import tempfile
+from typing import Any, Callable, Iterable, Optional, Tuple, Union
+from weakref import WeakKeyDictionary
+
+import scrapy
+from scrapy.http.response import Response
 
 from twisted.web import http
 from scrapy.utils.python import to_bytes, to_unicode
 from w3lib import html
 
 
-_baseurl_cache = weakref.WeakKeyDictionary()
+_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
 
 
-def get_base_url(response):
+def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
     """Return the base url of the given response, joined with the response url"""
     if response not in _baseurl_cache:
         text = response.text[0:4096]
@@ -23,10 +27,13 @@ def get_base_url(response):
     return _baseurl_cache[response]
 
 
-_metaref_cache = weakref.WeakKeyDictionary()
+_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary()
 
 
-def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
+def get_meta_refresh(
+    response: "scrapy.http.response.text.TextResponse",
+    ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'),
+) -> Union[Tuple[None, None], Tuple[float, str]]:
     """Parse the http-equiv refrsh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
@@ -35,14 +42,15 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
     return _metaref_cache[response]
 
 
-def response_status_message(status):
+def response_status_message(status: Union[bytes, float, int, str]) -> str:
     """Return status code plus status text descriptive message
     """
-    message = http.RESPONSES.get(int(status), "Unknown Status")
-    return f'{status} {to_unicode(message)}'
+    status_int = int(status)
+    message = http.RESPONSES.get(status_int, "Unknown Status")
+    return f'{status_int} {to_unicode(message)}'
 
 
-def response_httprepr(response):
+def response_httprepr(response: Response) -> bytes:
     """Return raw HTTP representation (as bytes) of the given response. This
     is provided only for reference, since it's not the exact stream of bytes
     that was received (that's not exposed by Twisted).
@@ -60,7 +68,10 @@ def response_httprepr(response):
     return b"".join(values)
 
 
-def open_in_browser(response, _openfunc=webbrowser.open):
+def open_in_browser(
+    response: Union["scrapy.http.response.html.HtmlResponse", "scrapy.http.response.text.TextResponse"],
+    _openfunc: Callable[[str], Any] = webbrowser.open,
+) -> Any:
     """Open the given response in a local web browser, populating the <base>
     tag for external links to work
     """
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 3e40acd69..b0c6a2424 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -9,14 +9,15 @@ and no performance penalty at all when disabled (as object_ref becomes just an
 alias to object in that case).
 """
 
-import weakref
-from time import time
-from operator import itemgetter
 from collections import defaultdict
+from operator import itemgetter
+from time import time
+from typing import DefaultDict
+from weakref import WeakKeyDictionary
 
 
 NoneType = type(None)
-live_refs = defaultdict(weakref.WeakKeyDictionary)
+live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)
 
 
 class object_ref:
diff --git a/setup.cfg b/setup.cfg
index 0c9f6b963..89b4ec57e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -10,54 +10,18 @@ follow_imports = skip
 
 # FIXME: remove the following sections once the issues are solved
 
-[mypy-scrapy]
-ignore_errors = True
-
-[mypy-scrapy.commands]
-ignore_errors = True
-
-[mypy-scrapy.commands.parse]
-ignore_errors = True
-
 [mypy-scrapy.downloadermiddlewares.httpproxy]
 ignore_errors = True
 
-[mypy-scrapy.contracts]
-ignore_errors = True
-
 [mypy-scrapy.interfaces]
 ignore_errors = True
 
-[mypy-scrapy.item]
-ignore_errors = True
-
-[mypy-scrapy.http.cookies]
-ignore_errors = True
-
-[mypy-scrapy.mail]
-ignore_errors = True
-
 [mypy-scrapy.pipelines.images]
 ignore_errors = True
 
 [mypy-scrapy.settings.default_settings]
 ignore_errors = True
 
-[mypy-scrapy.spidermiddlewares.referer]
-ignore_errors = True
-
-[mypy-scrapy.utils.httpobj]
-ignore_errors = True
-
-[mypy-scrapy.utils.request]
-ignore_errors = True
-
-[mypy-scrapy.utils.response]
-ignore_errors = True
-
-[mypy-scrapy.utils.trackref]
-ignore_errors = True
-
 [mypy-tests.mocks.dummydbm]
 ignore_errors = True
 

From db10aaf9eb858c8deb35e7ef96538549a8e36be0 Mon Sep 17 00:00:00 2001
From: gunadhya <6939749+gunadhya@users.noreply.github.com>
Date: Thu, 3 Dec 2020 15:26:36 +0530
Subject: [PATCH 239/716] Update links in installation guide (#4899)

---
 docs/conf.py           | 1 +
 docs/intro/install.rst | 4 +---
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 27d2b5dff..543507a46 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -283,6 +283,7 @@ coverage_ignore_pyobjects = [
 intersphinx_mapping = {
     'attrs': ('https://www.attrs.org/en/stable/', None),
     'coverage': ('https://coverage.readthedocs.io/en/stable', None),
+    'cryptography' : ('https://cryptography.io/en/latest/', None),
     'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
     'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
     'pytest': ('https://docs.pytest.org/en/latest', None),
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 3bfd3bc3b..73d7ede42 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -69,10 +69,9 @@ In case of any trouble related to these dependencies,
 please refer to their respective installation instructions:
 
 * `lxml installation`_
-* `cryptography installation`_
+* :doc:`cryptography installation <cryptography:installation>`
 
 .. _lxml installation: https://lxml.de/installation.html
-.. _cryptography installation: https://cryptography.io/en/latest/installation/
 
 
 .. _intro-using-virtualenv:
@@ -265,7 +264,6 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
 .. _cryptography: https://cryptography.io/en/latest/
 .. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/
 .. _setuptools: https://pypi.python.org/pypi/setuptools
-.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
 .. _homebrew: https://brew.sh/
 .. _zsh: https://www.zsh.org/
 .. _Scrapinghub: https://scrapinghub.com

From 0dff5781bc9089884e27b0a59c1e92fc111b2817 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Dec 2020 11:13:14 +0100
Subject: [PATCH 240/716] Blind attempt to fix the build of the
 cryptography-provided OpenSSL

---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index 6c39299bb..d3ff930c3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -146,6 +146,8 @@ commands =
 
 [testenv:pypy3-pinned]
 basepython = {[testenv:pypy3]basepython}
+before_install:
+  - sudo apt-get -y remove libssl-dev
 deps =
     {[pinned]deps}
     lxml==4.0.0

From 798a818cafb21eeb62790d905d43cec678d30693 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Dec 2020 13:35:40 +0100
Subject: [PATCH 241/716] Move apt-get command from Tox to Travis CI

---
 .travis.yml | 1 +
 tox.ini     | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f0eafd137..055304abe 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,6 +42,7 @@ matrix:
       python: 3.8
       dist: bionic
 install:
+  - apt-get -y remove libssl-dev
   - |
       if [[ ! -z "$PYPY_VERSION" ]]; then
         export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
diff --git a/tox.ini b/tox.ini
index d3ff930c3..6c39299bb 100644
--- a/tox.ini
+++ b/tox.ini
@@ -146,8 +146,6 @@ commands =
 
 [testenv:pypy3-pinned]
 basepython = {[testenv:pypy3]basepython}
-before_install:
-  - sudo apt-get -y remove libssl-dev
 deps =
     {[pinned]deps}
     lxml==4.0.0

From 1c1255a75d315cd535dbcc5689ddda9c80c79684 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Dec 2020 14:41:02 +0100
Subject: [PATCH 242/716] Use sudo for apt-get

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 055304abe..1a296c5e6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,7 +42,7 @@ matrix:
       python: 3.8
       dist: bionic
 install:
-  - apt-get -y remove libssl-dev
+  - sudo apt-get -y remove libssl-dev
   - |
       if [[ ! -z "$PYPY_VERSION" ]]; then
         export PYPY_VERSION="pypy$PYPY_VERSION-linux64"

From 45eb099ed1937a530208eb91e28e013d56b542ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 21 Dec 2020 15:37:02 +0100
Subject: [PATCH 243/716] =?UTF-8?q?Maybe=20it=E2=80=99s=20about=20having?=
 =?UTF-8?q?=20a=20newer=20libssl?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 1a296c5e6..880c8772b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,6 +23,7 @@ matrix:
     - env: TOXENV=asyncio-pinned
       python: 3.6.1
     - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0
+      dist: bionic
 
     - env: TOXENV=py
       python: 3.6
@@ -42,7 +43,6 @@ matrix:
       python: 3.8
       dist: bionic
 install:
-  - sudo apt-get -y remove libssl-dev
   - |
       if [[ ! -z "$PYPY_VERSION" ]]; then
         export PYPY_VERSION="pypy$PYPY_VERSION-linux64"

From b83a1a6fbfe12c205c7c57830b1d4a6fc1097be0 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 16 Dec 2020 18:02:47 -0300
Subject: [PATCH 244/716] Disable test under pypy

---
 tests/test_webclient.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a60181a3a..f935a8689 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -4,7 +4,10 @@ Tests borrowed from the twisted.web.client tests.
 """
 import os
 import shutil
+import sys
+from pkg_resources import parse_version
 
+import cryptography
 import OpenSSL.SSL
 from twisted.trial import unittest
 from twisted.web import server, static, util, resource
@@ -414,6 +417,8 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
         ).addCallback(self.assertEqual, to_bytes(s))
 
     def testPayloadDisabledCipher(self):
+        if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"):
+            self.skipTest("This does work in PyPy with cryptography<=2.3.1")
         s = "0123456789" * 10
         settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'})
         client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)

From 6dccf82eaa097f9e40e922d04a8b5ec9a0a9dbd7 Mon Sep 17 00:00:00 2001
From: Tim Gates <tim.gates@iress.com>
Date: Tue, 22 Dec 2020 07:49:13 +1100
Subject: [PATCH 245/716] docs: fix simple typo, wihout -> without

There is a small typo in scrapy/http/request/form.py.

Should read `without` rather than `wihout`.
---
 scrapy/http/request/form.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 2815303a2..7f267c800 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -160,7 +160,7 @@ def _select_value(ele, n, v):
     multiple = ele.multiple
     if v is None and not multiple:
         # Match browser behaviour on simple select tag without options selected
-        # And for select tags wihout options
+        # And for select tags without options
         o = ele.value_options
         return (n, o[0]) if o else (None, None)
     elif v is not None and multiple:

From 44a7ab5bf06be3bfdf4b4a304b24f544d2f833cd Mon Sep 17 00:00:00 2001
From: Kader DJEHAF <daek@online.fr>
Date: Wed, 30 Dec 2020 15:22:27 +0100
Subject: [PATCH 246/716] Fix warning: Expected type 'bool', got 'int' instead
 (#4940)

* Fix warning: Expected type 'bool', got 'int' instead

* Update defer.py
---
 scrapy/pipelines/media.py | 2 +-
 scrapy/utils/defer.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 0a12f3e2c..0c2ee6856 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -86,7 +86,7 @@ class MediaPipeline:
         info = self.spiderinfo
         requests = arg_to_iter(self.get_media_requests(item, info))
         dlist = [self._process_request(r, info, item) for r in requests]
-        dfd = DeferredList(dlist, consumeErrors=1)
+        dfd = DeferredList(dlist, consumeErrors=True)
         return dfd.addCallback(self.item_completed, item, info)
 
     def _process_request(self, request, info, item):
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 21ba02a0b..6db9cc117 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -105,7 +105,7 @@ def process_parallel(callbacks, input, *a, **kw):
     callbacks
     """
     dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks]
-    d = defer.DeferredList(dfds, fireOnOneErrback=1, consumeErrors=1)
+    d = defer.DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
     d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)
     return d
 

From e494a3f73318db76d7c56e65bc632cd5875b8825 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 31 Dec 2020 11:50:15 -0300
Subject: [PATCH 247/716] protocol attribute for h2 responses

---
 docs/topics/request-response.rst        | 2 +-
 scrapy/core/http2/stream.py             | 2 ++
 tests/test_downloader_handlers_http2.py | 7 +++++++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 98906992d..48f7f4a87 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -694,7 +694,7 @@ Response objects
     :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
 
     :param protocol: The protocol that was used to download the response.
-        For instance: "HTTP/1.0", "HTTP/1.1"
+        For instance: "HTTP/1.0", "HTTP/1.1", "h2"
     :type protocol: :class:`str`
 
     .. versionadded:: 2.0.0
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 3ae2e8db8..e345ca79a 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -15,6 +15,7 @@ from twisted.web.client import ResponseFailed
 from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
+from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
     from scrapy.core.http2.protocol import H2ClientProtocol
@@ -458,6 +459,7 @@ class Stream:
             request=self._request,
             certificate=self._protocol.metadata['certificate'],
             ip_address=self._protocol.metadata['ip_address'],
+            protocol=to_unicode(self._protocol.transport.negotiatedProtocol),
         )
 
         self._deferred_response.callback(response)
diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index 253646040..8f7f7aee0 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -22,6 +22,13 @@ class Https2TestCase(Https11TestCase):
     download_handler_cls = H2DownloadHandler
     HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
 
+    def test_protocol(self):
+        request = Request(self.getURL("host"), method="GET")
+        d = self.download_request(request, Spider("foo"))
+        d.addCallback(lambda r: r.protocol)
+        d.addCallback(self.assertEqual, "h2")
+        return d
+
     @defer.inlineCallbacks
     def test_download_with_maxsize_very_large_file(self):
         with mock.patch('scrapy.core.http2.stream.logger') as logger:

From 80db569aeae3e05480ed228c95332c17f67e81e4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Fri, 1 Jan 2021 19:13:39 -0300
Subject: [PATCH 248/716] Migrate CI to GitHub actions (#4924)

---
 .github/workflows/checks.yml        | 38 ++++++++++++++
 .github/workflows/main.yml          | 31 ------------
 .github/workflows/publish.yml       | 31 ++++++++++++
 .github/workflows/tests-macos.yml   | 25 ++++++++++
 .github/workflows/tests-ubuntu.yml  | 69 ++++++++++++++++++++++++++
 .github/workflows/tests-windows.yml | 32 ++++++++++++
 .travis.yml                         | 77 -----------------------------
 README.rst                          | 14 ++++--
 tests/requirements-py3.txt          |  2 +-
 tox.ini                             | 24 ++-------
 10 files changed, 211 insertions(+), 132 deletions(-)
 create mode 100644 .github/workflows/checks.yml
 delete mode 100644 .github/workflows/main.yml
 create mode 100644 .github/workflows/publish.yml
 create mode 100644 .github/workflows/tests-macos.yml
 create mode 100644 .github/workflows/tests-ubuntu.yml
 create mode 100644 .github/workflows/tests-windows.yml
 delete mode 100644 .travis.yml

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
new file mode 100644
index 000000000..2748bf5fe
--- /dev/null
+++ b/.github/workflows/checks.yml
@@ -0,0 +1,38 @@
+name: Checks
+on: [push, pull_request]
+
+jobs:
+  checks:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        include:
+        - python-version: 3.8
+          env:
+            TOXENV: security
+        - python-version: 3.8
+          env:
+            TOXENV: flake8
+        - python-version: 3.8
+          env:
+            TOXENV: pylint
+        - python-version: 3.8
+          env:
+            TOXENV: typing
+        - python-version: 3.7  # Keep in sync with .readthedocs.yml
+          env:
+            TOXENV: docs
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Run check
+      env: ${{ matrix.env }}
+      run: |
+        pip install -U tox
+        tox
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
deleted file mode 100644
index 28771216c..000000000
--- a/.github/workflows/main.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-name: Run test suite
-on: [push, pull_request]
-
-jobs:
-  test-windows:
-    name: "Windows Tests"
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [windows-latest]
-        python-version: [3.7, 3.8]
-        env: [TOXENV: py]
-        include:
-        - os: windows-latest
-          python-version: 3.6
-          env:
-            TOXENV: windows-pinned
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Run test suite
-      env: ${{ matrix.env }}
-      run: |
-        pip install -U tox twine wheel codecov
-        tox
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
new file mode 100644
index 000000000..aec6b8696
--- /dev/null
+++ b/.github/workflows/publish.yml
@@ -0,0 +1,31 @@
+name: Publish
+on: [push]
+
+jobs:
+  publish:
+    runs-on: ubuntu-18.04
+    if: startsWith(github.event.ref, 'refs/tags/')
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Check Tag
+      id: check-release-tag
+      run: |
+        if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
+          echo ::set-output name=release_tag::true
+        fi
+
+    - name: Publish to PyPI
+      if: steps.check-release-tag.outputs.release_tag == 'true'
+      run: |
+        pip install --upgrade setuptools wheel twine
+        python setup.py sdist bdist_wheel
+        export TWINE_USERNAME=__token__
+        export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
+        twine upload dist/*
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
new file mode 100644
index 000000000..51d27c405
--- /dev/null
+++ b/.github/workflows/tests-macos.yml
@@ -0,0 +1,25 @@
+name: macOS
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: macos-10.15
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Run tests
+      run: |
+        pip install -U tox
+        tox -e py
+
+    - name: Upload coverage report
+      run: bash <(curl -s https://codecov.io/bash)
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
new file mode 100644
index 000000000..89c0334e2
--- /dev/null
+++ b/.github/workflows/tests-ubuntu.yml
@@ -0,0 +1,69 @@
+name: Ubuntu
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        include:
+        - python-version: 3.7
+          env:
+            TOXENV: py
+        - python-version: 3.8
+          env:
+            TOXENV: py
+        - python-version: pypy3
+          env:
+            TOXENV: pypy3
+            PYPY_VERSION: 3.6-v7.3.1
+
+        # pinned deps
+        - python-version: 3.6.12
+          env:
+            TOXENV: pinned
+        - python-version: 3.6.12
+          env:
+            TOXENV: asyncio-pinned
+        - python-version: pypy3
+          env:
+            TOXENV: pypy3-pinned
+            PYPY_VERSION: 3.6-v7.2.0
+
+        # extras
+        - python-version: 3.8
+          env:
+            TOXENV: extra-deps
+        - python-version: 3.8
+          env:
+            TOXENV: asyncio
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install system libraries
+      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')
+      run: |
+        sudo apt-get update
+        sudo apt-get install libxml2-dev libxslt-dev
+
+    - name: Run tests
+      env: ${{ matrix.env }}
+      run: |
+        if [[ ! -z "$PYPY_VERSION" ]]; then
+          export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
+          wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
+          tar -jxf ${PYPY_VERSION}.tar.bz2
+          $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
+          source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
+        fi
+        pip install -U tox
+        tox
+
+    - name: Upload coverage report
+      run: bash <(curl -s https://codecov.io/bash)
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
new file mode 100644
index 000000000..ed2e4075d
--- /dev/null
+++ b/.github/workflows/tests-windows.yml
@@ -0,0 +1,32 @@
+name: Windows
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        include:
+        - python-version: 3.6
+          env:
+            TOXENV: windows-pinned
+        - python-version: 3.7
+          env:
+            TOXENV: py
+        - python-version: 3.8
+          env:
+            TOXENV: py
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Run tests
+      env: ${{ matrix.env }}
+      run: |
+        pip install -U tox
+        tox
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 880c8772b..000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,77 +0,0 @@
-language: python
-dist: xenial
-branches:
-  only:
-    - master
-    - /^\d\.\d+$/
-    - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
-matrix:
-  include:
-    - env: TOXENV=security
-      python: 3.8
-    - env: TOXENV=flake8
-      python: 3.8
-    - env: TOXENV=pylint
-      python: 3.8
-    - env: TOXENV=docs
-      python: 3.7  # Keep in sync with .readthedocs.yml
-    - env: TOXENV=typing
-      python: 3.8
-
-    - env: TOXENV=pinned
-      python: 3.6.1
-    - env: TOXENV=asyncio-pinned
-      python: 3.6.1
-    - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0
-      dist: bionic
-
-    - env: TOXENV=py
-      python: 3.6
-    - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
-      dist: bionic
-
-    - env: TOXENV=py
-      python: 3.7
-
-    - env: TOXENV=py PYPI_RELEASE_JOB=true
-      python: 3.8
-      dist: bionic
-    - env: TOXENV=extra-deps
-      python: 3.8
-      dist: bionic
-    - env: TOXENV=asyncio
-      python: 3.8
-      dist: bionic
-install:
-  - |
-      if [[ ! -z "$PYPY_VERSION" ]]; then
-        export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
-        wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
-        tar -jxf ${PYPY_VERSION}.tar.bz2
-        virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
-        source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-      fi
-  - pip install -U tox twine wheel codecov
-
-script: tox
-after_success:
-  - codecov
-notifications:
-  irc:
-    use_notice: true
-    skip_join: true
-    channels:
-    - irc.freenode.org#scrapy
-cache:
-  directories:
-    - $HOME/.cache/pip
-deploy:
-  provider: pypi
-  distributions: "sdist bdist_wheel"
-  user: scrapy
-  password:
-    secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA=
-  on:
-    tags: true
-    repo: scrapy/scrapy
-    condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
diff --git a/README.rst b/README.rst
index a8f2ba52b..9418d270f 100644
--- a/README.rst
+++ b/README.rst
@@ -10,9 +10,17 @@ Scrapy
    :target: https://pypi.python.org/pypi/Scrapy
    :alt: Supported Python Versions
 
-.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
-   :target: https://travis-ci.org/scrapy/scrapy
-   :alt: Build Status
+.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg
+   :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
+   :alt: Ubuntu
+
+.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
+   :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
+   :alt: macOS
+
+.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
+   :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
+   :alt: Windows
 
 .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
    :target: https://pypi.python.org/pypi/Scrapy
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 68b856a88..a86c4ae4f 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -14,6 +14,6 @@ uvloop; platform_system != "Windows"
 # optional for shell wrapper tests
 bpython
 brotlipy  # optional for HTTP compress downloader middleware tests
-zstandard  # optional for HTTP compress downloader middleware tests
+zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 ipython
 pywin32; sys_platform == "win32"
diff --git a/tox.ini b/tox.ini
index 6c39299bb..e70aef2d2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,7 +13,7 @@ deps =
     -rtests/requirements-py3.txt
     # mitmproxy does not support PyPy
     # mitmproxy does not support Windows when running Python < 3.7
-    mitmproxy; python_version >= '3.7' and implementation_name != 'pypy'
+    mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy'
     mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # Extras
     botocore>=1.4.87
@@ -25,9 +25,9 @@ passenv =
     GCS_TEST_FILE_URI
     GCS_PROJECT_ID
 #allow tox virtualenv to upgrade pip/wheel/setuptools
-download = true 
+download = true
 commands =
-    py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
+    py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
 
 [testenv:typing]
 basepython = python3
@@ -87,10 +87,6 @@ deps =
     botocore==1.4.87
     google-cloud-storage==1.29.0
     Pillow==4.0.0
-install_command =
-    # --use-feature=2020-resolver is required, otherwise the latest verion of
-    # Twisted gets installed.
-    pip install --use-feature=2020-resolver {opts} {packages}
 setenv =
     _SCRAPY_PINNED=true
 
@@ -99,11 +95,9 @@ deps =
     {[pinned]deps}
     lxml==3.5.0
     PyDispatcher==2.0.5
-install_command =
-    {[pinned]install_command}
 setenv =
     {[pinned]setenv}
-    
+
 [testenv:windows-pinned]
 basepython = python3
 deps =
@@ -112,8 +106,6 @@ deps =
     # not need to build lxml from sources in a CI Windows job:
     lxml==3.8.0
     PyDispatcher==2.0.5
-install_command =
-    {[pinned]install_command}
 setenv =
     {[pinned]setenv}
 
@@ -122,10 +114,6 @@ deps =
     {[testenv]deps}
     reppy
     robotexclusionrulesparser
-install_command =
-    # Test --use-feature=2020-resolver for the latest version of all
-    # dependencies.
-    pip install --use-feature=2020-resolver {opts} {packages}
 
 [testenv:asyncio]
 commands =
@@ -133,8 +121,6 @@ commands =
 
 [testenv:asyncio-pinned]
 deps = {[testenv:pinned]deps}
-install_command =
-    {[pinned]install_command}
 commands = {[testenv:asyncio]commands}
 setenv =
     {[pinned]setenv}
@@ -150,8 +136,6 @@ deps =
     {[pinned]deps}
     lxml==4.0.0
     PyPyDispatcher==2.1.0
-install_command =
-    {[pinned]install_command}
 commands = {[testenv:pypy3]commands}
 setenv =
     {[pinned]setenv}

From 0a1e2fefab04984a0f3c2b470346fd9f2ffde65b Mon Sep 17 00:00:00 2001
From: M Ikram Ullah Khan <44160462+IkramKhanNiazi@users.noreply.github.com>
Date: Mon, 4 Jan 2021 18:30:23 +0500
Subject: [PATCH 249/716] Docs: fix typo in news.rst (#4942)

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 0391506c4..d9fe897ad 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -2428,7 +2428,7 @@ Bug fixes
 - Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`).
 - Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`).
 - Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``,
-  ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
+  ``DOWNLOADER_MIDDLEWARES``, ``EXTENSIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
 
 Documentation
 ~~~~~~~~~~~~~

From 6e7ae789f9d0b62ef5e07d16fab062ef7bdfa660 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 14 Jan 2021 11:59:38 +0100
Subject: [PATCH 250/716] Reuse the text from https://scrapy.org/

---
 README.rst | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/README.rst b/README.rst
index 19faa9a87..551a06f6b 100644
--- a/README.rst
+++ b/README.rst
@@ -34,11 +34,10 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
 crawl websites and extract structured data from their pages. It can be used for
 a wide range of purposes, from data mining to monitoring and automated testing.
 
-Scrapy has contributions from `many users`_ (thanks everyone!) and is sponsored
-by `Scrapinghub Ltd`_.
+Scrapy is maintained by `Scrapinghub`_ and `many other contributors`_.
 
-.. _many users: https://github.com/scrapy/scrapy/graphs/contributors
-.. _Scrapinghub Ltd: https://www.scrapinghub.com/
+.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors
+.. _Scrapinghub: https://www.scrapinghub.com/
 
 Check the Scrapy homepage at https://scrapy.org for more information,
 including a list of features.

From f30f53b3cc958770406628411db3d93c925db59e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 2 Feb 2021 15:03:20 +0100
Subject: [PATCH 251/716] =?UTF-8?q?Scrapinghub=20=E2=86=92=20Zyte?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 AUTHORS                                   |  4 +--
 CODE_OF_CONDUCT.md                        |  2 +-
 README.rst                                |  7 ++---
 docs/intro/install.rst                    |  1 -
 docs/topics/deploy.rst                    | 32 +++++++++++------------
 docs/topics/logging.rst                   |  4 +--
 docs/topics/practices.rst                 |  6 ++---
 docs/topics/selectors.rst                 |  4 +--
 scrapy/core/downloader/handlers/http11.py | 13 +++++----
 9 files changed, 38 insertions(+), 35 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index bcaa1ecd3..9706adf42 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -1,8 +1,8 @@
 Scrapy was brought to life by Shane Evans while hacking a scraping framework
 prototype for Mydeco (mydeco.com). It soon became maintained, extended and
 improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to
-bootstrap the project. In mid-2011, Scrapinghub became the new official
-maintainer.
+bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new
+official maintainer.
 
 Here is the list of the primary authors & contributors:
 
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index d1cd3e517..652460383 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at opensource@scrapinghub.com. All
+reported by contacting the project team at opensource@zyte.com. All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.
diff --git a/README.rst b/README.rst
index bbe346522..5750e2c0f 100644
--- a/README.rst
+++ b/README.rst
@@ -42,10 +42,11 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
 crawl websites and extract structured data from their pages. It can be used for
 a wide range of purposes, from data mining to monitoring and automated testing.
 
-Scrapy is maintained by `Scrapinghub`_ and `many other contributors`_.
+Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other
+contributors`_.
 
 .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors
-.. _Scrapinghub: https://www.scrapinghub.com/
+.. _Zyte: https://www.zyte.com/
 
 Check the Scrapy homepage at https://scrapy.org for more information,
 including a list of features.
@@ -95,7 +96,7 @@ Please note that this project is released with a Contributor Code of Conduct
 (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
 
 By participating in this project you agree to abide by its terms.
-Please report unacceptable behavior to opensource@scrapinghub.com.
+Please report unacceptable behavior to opensource@zyte.com.
 
 Companies using Scrapy
 ======================
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 73d7ede42..bf919ce25 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -266,7 +266,6 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
 .. _setuptools: https://pypi.python.org/pypi/setuptools
 .. _homebrew: https://brew.sh/
 .. _zsh: https://www.zsh.org/
-.. _Scrapinghub: https://scrapinghub.com
 .. _Anaconda: https://docs.anaconda.com/anaconda/
 .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
 .. _conda-forge: https://conda-forge.org/
diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst
index 361914a29..961d6dc01 100644
--- a/docs/topics/deploy.rst
+++ b/docs/topics/deploy.rst
@@ -14,7 +14,7 @@ spiders come in.
 Popular choices for deploying Scrapy spiders are:
 
 * :ref:`Scrapyd <deploy-scrapyd>` (open source)
-* :ref:`Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
+* :ref:`Zyte Scrapy Cloud <deploy-scrapy-cloud>` (cloud-based)
 
 .. _deploy-scrapyd:
 
@@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers.
 
 .. _deploy-scrapy-cloud:
 
-Deploying to Scrapy Cloud
-=========================
+Deploying to Zyte Scrapy Cloud
+==============================
 
-`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_,
-the company behind Scrapy.
+`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company
+behind Scrapy.
 
-Scrapy Cloud removes the need to setup and monitor servers
-and provides a nice UI to manage spiders and review scraped items,
-logs and stats.
+Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a
+nice UI to manage spiders and review scraped items, logs and stats.
 
-To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool.
-Please refer to the `Scrapy Cloud documentation`_ for more information.
+To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line
+tool.
+Please refer to the `Zyte Scrapy Cloud documentation`_ for more information.
 
-Scrapy Cloud is compatible with Scrapyd and one can switch between
+Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between
 them as needed - the configuration is read from the ``scrapy.cfg`` file
 just like ``scrapyd-deploy``.
 
-.. _Scrapyd: https://github.com/scrapy/scrapyd
 .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
-.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud
+.. _Scrapyd: https://github.com/scrapy/scrapyd
 .. _scrapyd-client: https://github.com/scrapy/scrapyd-client
-.. _shub: https://doc.scrapinghub.com/shub.html
 .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html
-.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html
-.. _Scrapinghub: https://scrapinghub.com/
+.. _shub: https://shub.readthedocs.io/en/latest/
+.. _Zyte: https://zyte.com/
+.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/
+.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 55065a1a3..c3445d40e 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -101,7 +101,7 @@ instance, which can be accessed and used like this::
     class MySpider(scrapy.Spider):
 
         name = 'myspider'
-        start_urls = ['https://scrapinghub.com']
+        start_urls = ['https://scrapy.org']
 
         def parse(self, response):
             self.logger.info('Parse function called on %s', response.url)
@@ -117,7 +117,7 @@ Python logger you want. For example::
     class MySpider(scrapy.Spider):
 
         name = 'myspider'
-        start_urls = ['https://scrapinghub.com']
+        start_urls = ['https://scrapy.org']
 
         def parse(self, response):
             logger.info('Parse function called on %s', response.url)
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index cf1de1bd1..502fd5fcd 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -63,7 +63,7 @@ project as example.
     process = CrawlerProcess(get_project_settings())
 
     # 'followall' is the name of one of the spiders of the project.
-    process.crawl('followall', domain='scrapinghub.com')
+    process.crawl('followall', domain='scrapy.org')
     process.start() # the script will block here until the crawling is finished
 
 There's another Scrapy utility that provides more control over the crawling
@@ -244,7 +244,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
   super proxy that you can attach your own proxies to.
 * use a highly distributed downloader that circumvents bans internally, so you
   can just focus on parsing clean pages. One example of such downloaders is
-  `Crawlera`_
+  `Zyte Smart Proxy Manager`_
 
 If you are still unable to prevent your bot getting banned, consider contacting
 `commercial support`_.
@@ -254,5 +254,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
 .. _ProxyMesh: https://proxymesh.com/
 .. _Google cache: http://www.googleguide.com/cached_pages.html
 .. _testspiders: https://github.com/scrapinghub/testspiders
-.. _Crawlera: https://scrapinghub.com/crawlera
 .. _scrapoxy: https://scrapoxy.io/
+.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index b576fde91..c7ec2e0cc 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -464,10 +464,10 @@ effectively. If you are not much familiar with XPath yet,
 you may want to take a look first at this `XPath tutorial`_.
 
 .. note::
-    Some of the tips are based on `this post from ScrapingHub's blog`_.
+    Some of the tips are based on `this post from Zyte's blog`_.
 
 .. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
-.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
+.. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/
 
 
 .. _topics-selectors-relative-xpaths:
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index a0fd837b1..513df2de9 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -303,11 +303,14 @@ class ScrapyAgent:
             proxyHost = to_unicode(proxyHost)
             omitConnectTunnel = b'noconnect' in proxyParams
             if omitConnectTunnel:
-                warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
-                              "If you use Crawlera, it doesn't require this mode anymore, "
-                              "so you should update scrapy-crawlera to 1.3.0+ "
-                              "and remove '?noconnect' from the Crawlera URL.",
-                              ScrapyDeprecationWarning)
+                warnings.warn(
+                    "Using HTTPS proxies in the noconnect mode is deprecated. "
+                    "If you use Zyte Smart Proxy Manager (formerly Crawlera), "
+                    "it doesn't require this mode anymore, so you should "
+                    "update scrapy-crawlera to 1.3.0+ and remove '?noconnect' "
+                    "from the Zyte Smart Proxy Manager URL.",
+                    ScrapyDeprecationWarning,
+                )
             if scheme == b'https' and not omitConnectTunnel:
                 proxyAuth = request.headers.get(b'Proxy-Authorization', None)
                 proxyConf = (proxyHost, proxyPort, proxyAuth)

From 2ce8e0c74200481f8249bcee0f2a8a249e8fd58c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 3 Feb 2021 09:09:53 +0100
Subject: [PATCH 252/716] Document the (hard-coded) maximum HTTP/2 frame size
 accepted from servers

---
 docs/topics/settings.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index b948dbfde..c7b59d582 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -689,10 +689,15 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
     Scrapy currently does not support HTTP/2 Cleartext (h2c) since none
     of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_).
 
+    Also, Scrapy does not currently support specifying a maximum `frame size`_
+    larger than the default value, 16384. Connections to servers that send a
+    larger frame will fail.
+
 .. warning:: HTTP/2 support in Scrapy is experimental, and not yet recommended
              for production environments. Future Scrapy versions may introduce
              related changes without a deprecation period or warning.
 
+.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2
 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
 
 .. setting:: DOWNLOAD_TIMEOUT

From d1024566d85e52f71f5591e53a3a9ee4867148ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 3 Feb 2021 09:13:45 +0100
Subject: [PATCH 253/716] =?UTF-8?q?setup.py:=20Twisted=20=E2=86=92=20Twist?=
 =?UTF-8?q?ed[http2]?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index a95014c98..d1d6cc4b6 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support():
 
 
 install_requires = [
-    'Twisted>=17.9.0',
+    'Twisted[http2]>=17.9.0',
     'cryptography>=2.0',
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',

From 536e749eccbdf479c9849b1868a5d95aa1210225 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 3 Feb 2021 09:22:02 +0100
Subject: [PATCH 254/716] HTTP/2: remove verbose protocol-handling logging

---
 scrapy/core/http2/protocol.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index d8d0974b8..36a51b898 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -218,7 +218,6 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self.setTimeout(self.IDLE_TIMEOUT)
 
         destination = self.transport.getPeer()
-        logger.debug('Connection made to {}'.format(destination))
         self.metadata['ip_address'] = ipaddress.ip_address(destination.host)
 
         # Initiate H2 Connection
@@ -347,7 +346,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             elif isinstance(event, SettingsAcknowledged):
                 self.settings_acknowledged(event)
             elif isinstance(event, UnknownFrameReceived):
-                logger.debug('UnknownFrameReceived: frame={}'.format(event.frame))
+                logger.warning(f'Unknown frame received: {event.frame}')
 
     # Event handler functions starts here
     def connection_terminated(self, event: ConnectionTerminated) -> None:
@@ -357,15 +356,19 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
     def data_received(self, event: DataReceived) -> None:
         try:
-            self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length)
+            stream = self.streams[event.stream_id]
         except KeyError:
-            logger.debug(f'Ignoring server-initiated event {event}')
+            pass  # We ignore server-initiated events
+        else:
+            stream.receive_data(event.data, event.flow_controlled_length)
 
     def response_received(self, event: ResponseReceived) -> None:
         try:
-            self.streams[event.stream_id].receive_headers(event.headers)
+            stream = self.streams[event.stream_id]
         except KeyError:
-            logger.debug(f'Ignoring server-initiated event {event}')
+            pass  # We ignore server-initiated events
+        else:
+            stream.receive_headers(event.headers)
 
     def settings_acknowledged(self, event: SettingsAcknowledged) -> None:
         self.metadata['settings_acknowledged'] = True
@@ -381,7 +384,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         try:
             stream = self.pop_stream(event.stream_id)
         except KeyError:
-            logger.debug(f'Ignoring server-initiated event {event}')
+            pass  # We ignore server-initiated events
         else:
             stream.close(StreamCloseReason.ENDED, from_protocol=True)
 
@@ -389,7 +392,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         try:
             stream = self.pop_stream(event.stream_id)
         except KeyError:
-            logger.debug(f'Ignoring server-initiated event {event}')
+            pass  # We ignore server-initiated events
         else:
             stream.close(StreamCloseReason.RESET, from_protocol=True)
 

From 1a7bde0d8e142acc06a534f6daf8910fde5de06d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 3 Feb 2021 10:55:11 +0100
Subject: [PATCH 255/716] Document that HTTP/2 server pushes are ignored

---
 docs/topics/settings.rst | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c7b59d582..05cb4a855 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -684,21 +684,28 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
         'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
     }
 
+.. warning::
+
+    HTTP/2 support in Scrapy is experimental, and not yet recommended for
+    production environments. Future Scrapy versions may introduce related
+    changes without a deprecation period or warning.
+
 .. note::
 
-    Scrapy currently does not support HTTP/2 Cleartext (h2c) since none
-    of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_).
+    Known limitations of the current HTTP/2 implementation of Scrapy include:
 
-    Also, Scrapy does not currently support specifying a maximum `frame size`_
-    larger than the default value, 16384. Connections to servers that send a
-    larger frame will fail.
+    -   No support for HTTP/2 Cleartext (h2c), since no major browser supports
+        HTTP/2 unencrypted (refer `http2 faq`_).
 
-.. warning:: HTTP/2 support in Scrapy is experimental, and not yet recommended
-             for production environments. Future Scrapy versions may introduce
-             related changes without a deprecation period or warning.
+    -   No setting to specify a maximum `frame size`_ larger than the default
+        value, 16384. Connections to servers that send a larger frame will
+        fail.
+
+    -   No support for `server pushes`_, which are ignored.
 
 .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2
 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
+.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2
 
 .. setting:: DOWNLOAD_TIMEOUT
 

From 1773eaf5dc2330464cf769b7d874f24033f69707 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 3 Feb 2021 11:43:18 +0100
Subject: [PATCH 256/716] Move lists closer to their introducing paragraph

---
 docs/_static/custom.css      |  10 +++
 docs/conf.py                 |   5 +-
 docs/topics/exceptions.rst   |   8 +-
 docs/topics/feed-exports.rst | 139 ++++++++++++++++++-----------------
 docs/topics/selectors.rst    |  14 ++--
 docs/topics/shell.rst        |  45 ++++++------
 6 files changed, 121 insertions(+), 100 deletions(-)
 create mode 100644 docs/_static/custom.css

diff --git a/docs/_static/custom.css b/docs/_static/custom.css
new file mode 100644
index 000000000..64f16939c
--- /dev/null
+++ b/docs/_static/custom.css
@@ -0,0 +1,10 @@
+/* Move lists closer to their introducing paragraph */
+.rst-content .section ol p, .rst-content .section ul p {
+    margin-bottom: 0px;
+}
+.rst-content p + ol, .rst-content p + ul {
+    margin-top: -18px; /* Compensates margin-top: 24px of p  */
+}
+.rst-content dl p + ol, .rst-content dl p + ul {
+    margin-top: -6px; /* Compensates margin-top: 12px of p  */
+}
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index 543507a46..406c4d94a 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -122,7 +122,6 @@ html_theme = 'sphinx_rtd_theme'
 import sphinx_rtd_theme
 html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
-
 # The style sheet to use for HTML and HTML Help pages. A file of that name
 # must exist either in Sphinx' static/ path, or in one of the custom paths
 # given in html_static_path.
@@ -183,6 +182,10 @@ html_copy_source = True
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'Scrapydoc'
 
+html_css_files = [
+    'custom.css',
+]
+
 
 # Options for LaTeX output
 # ------------------------
diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 583a50ab8..e5264d641 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -64,10 +64,10 @@ NotConfigured
 This exception can be raised by some components to indicate that they will
 remain disabled. Those components include:
 
- * Extensions
- * Item pipelines
- * Downloader middlewares
- * Spider middlewares
+-   Extensions
+-   Item pipelines
+-   Downloader middlewares
+-   Spider middlewares
 
 The exception must be raised in the component's ``__init__`` method.
 
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 843ed25f9..e772a461c 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -21,10 +21,10 @@ Serialization formats
 For serializing the scraped data, the feed exports use the :ref:`Item exporters
 <topics-exporters>`. These formats are supported out of the box:
 
- * :ref:`topics-feed-format-json`
- * :ref:`topics-feed-format-jsonlines`
- * :ref:`topics-feed-format-csv`
- * :ref:`topics-feed-format-xml`
+-   :ref:`topics-feed-format-json`
+-   :ref:`topics-feed-format-jsonlines`
+-   :ref:`topics-feed-format-csv`
+-   :ref:`topics-feed-format-xml`
 
 But you can also extend the supported format through the
 :setting:`FEED_EXPORTERS` setting.
@@ -34,54 +34,58 @@ But you can also extend the supported format through the
 JSON
 ----
 
- * Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
- * Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
- * See :ref:`this warning <json-with-large-data>` if you're using JSON with
-   large feeds.
+-   Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
+
+-   Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
+
+-   See :ref:`this warning <json-with-large-data>` if you're using JSON with
+    large feeds.
 
 .. _topics-feed-format-jsonlines:
 
 JSON lines
 ----------
 
- * Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
- * Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
+-   Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
+-   Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
 
 .. _topics-feed-format-csv:
 
 CSV
 ---
 
- * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
- * Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
- * To specify columns to export and their order use
-   :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
-   option, but it is important for CSV because unlike many other export
-   formats CSV uses a fixed header.
+-   Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
+
+-   Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
+
+-   To specify columns to export and their order use
+    :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
+    option, but it is important for CSV because unlike many other export
+    formats CSV uses a fixed header.
 
 .. _topics-feed-format-xml:
 
 XML
 ---
 
- * Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
- * Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
+-   Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
+-   Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
 
 .. _topics-feed-format-pickle:
 
 Pickle
 ------
 
- * Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
- * Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
+-   Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
+-   Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
 
 .. _topics-feed-format-marshal:
 
 Marshal
 -------
 
- * Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
- * Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
+-   Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
+-   Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
 
 
 .. _topics-feed-storage:
@@ -95,11 +99,11 @@ storage backend types which are defined by the URI scheme.
 
 The storages backends supported out of the box are:
 
- * :ref:`topics-feed-storage-fs`
- * :ref:`topics-feed-storage-ftp`
- * :ref:`topics-feed-storage-s3` (requires botocore_)
- * :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
- * :ref:`topics-feed-storage-stdout`
+-   :ref:`topics-feed-storage-fs`
+-   :ref:`topics-feed-storage-ftp`
+-   :ref:`topics-feed-storage-s3` (requires botocore_)
+-   :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
+-   :ref:`topics-feed-storage-stdout`
 
 Some storage backends may be unavailable if the required external libraries are
 not available. For example, the S3 backend is only available if the botocore_
@@ -114,8 +118,8 @@ Storage URI parameters
 The storage URI can also contain parameters that get replaced when the feed is
 being created. These parameters are:
 
- * ``%(time)s`` - gets replaced by a timestamp when the feed is being created
- * ``%(name)s`` - gets replaced by the spider name
+-   ``%(time)s`` - gets replaced by a timestamp when the feed is being created
+-   ``%(name)s`` - gets replaced by the spider name
 
 Any other named parameter gets replaced by the spider attribute of the same
 name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id``
@@ -123,13 +127,13 @@ attribute the moment the feed is being created.
 
 Here are some examples to illustrate:
 
- * Store in FTP using one directory per spider:
+-   Store in FTP using one directory per spider:
 
-   * ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json``
+    -   ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json``
 
- * Store in S3 using one directory per spider:
+-   Store in S3 using one directory per spider:
 
-   * ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
+    -   ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
 
 
 .. _topics-feed-storage-backends:
@@ -144,9 +148,9 @@ Local filesystem
 
 The feeds are stored in the local filesystem.
 
- * URI scheme: ``file``
- * Example URI: ``file:///tmp/export.csv``
- * Required external libraries: none
+-   URI scheme: ``file``
+-   Example URI: ``file:///tmp/export.csv``
+-   Required external libraries: none
 
 Note that for the local filesystem storage (only) you can omit the scheme if
 you specify an absolute path like ``/tmp/export.csv``. This only works on Unix
@@ -159,9 +163,9 @@ FTP
 
 The feeds are stored in a FTP server.
 
- * URI scheme: ``ftp``
- * Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
- * Required external libraries: none
+-   URI scheme: ``ftp``
+-   Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
+-   Required external libraries: none
 
 FTP supports two different connection modes: `active or passive
 <https://stackoverflow.com/a/1699163>`_. Scrapy uses the passive connection
@@ -178,23 +182,25 @@ S3
 
 The feeds are stored on `Amazon S3`_.
 
- * URI scheme: ``s3``
- * Example URIs:
+-   URI scheme: ``s3``
 
-   * ``s3://mybucket/path/to/export.csv``
-   * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
+-   Example URIs:
 
- * Required external libraries: `botocore`_ >= 1.4.87
+    -   ``s3://mybucket/path/to/export.csv``
+
+    -   ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
+
+-   Required external libraries: `botocore`_ >= 1.4.87
 
 The AWS credentials can be passed as user/password in the URI, or they can be
 passed through the following settings:
 
- * :setting:`AWS_ACCESS_KEY_ID`
- * :setting:`AWS_SECRET_ACCESS_KEY`
+-   :setting:`AWS_ACCESS_KEY_ID`
+-   :setting:`AWS_SECRET_ACCESS_KEY`
 
 You can also define a custom ACL for exported feeds using this setting:
 
- * :setting:`FEED_STORAGE_S3_ACL`
+-   :setting:`FEED_STORAGE_S3_ACL`
 
 This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
 
@@ -208,19 +214,20 @@ Google Cloud Storage (GCS)
 
 The feeds are stored on `Google Cloud Storage`_.
 
- * URI scheme: ``gs``
- * Example URIs:
+-   URI scheme: ``gs``
 
-   * ``gs://mybucket/path/to/export.csv``
+-   Example URIs:
 
- * Required external libraries: `google-cloud-storage`_.
+    -   ``gs://mybucket/path/to/export.csv``
+
+-   Required external libraries: `google-cloud-storage`_.
 
 For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
 
 You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
 
- * :setting:`FEED_STORAGE_GCS_ACL`
- * :setting:`GCS_PROJECT_ID`
+-   :setting:`FEED_STORAGE_GCS_ACL`
+-   :setting:`GCS_PROJECT_ID`
 
 This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
 
@@ -234,9 +241,9 @@ Standard output
 
 The feeds are written to the standard output of the Scrapy process.
 
- * URI scheme: ``stdout``
- * Example URI: ``stdout:``
- * Required external libraries: none
+-   URI scheme: ``stdout``
+-   Example URI: ``stdout:``
+-   Required external libraries: none
 
 
 .. _delayed-file-delivery:
@@ -264,16 +271,16 @@ Settings
 
 These are the settings used for configuring the feed exports:
 
- * :setting:`FEEDS` (mandatory)
- * :setting:`FEED_EXPORT_ENCODING`
- * :setting:`FEED_STORE_EMPTY`
- * :setting:`FEED_EXPORT_FIELDS`
- * :setting:`FEED_EXPORT_INDENT`
- * :setting:`FEED_STORAGES`
- * :setting:`FEED_STORAGE_FTP_ACTIVE`
- * :setting:`FEED_STORAGE_S3_ACL`
- * :setting:`FEED_EXPORTERS`
- * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
+-   :setting:`FEEDS` (mandatory)
+-   :setting:`FEED_EXPORT_ENCODING`
+-   :setting:`FEED_STORE_EMPTY`
+-   :setting:`FEED_EXPORT_FIELDS`
+-   :setting:`FEED_EXPORT_INDENT`
+-   :setting:`FEED_STORAGES`
+-   :setting:`FEED_STORAGE_FTP_ACTIVE`
+-   :setting:`FEED_STORAGE_S3_ACL`
+-   :setting:`FEED_EXPORTERS`
+-   :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index c7ec2e0cc..9caba5ee5 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -8,14 +8,14 @@ When you're scraping web pages, the most common task you need to perform is
 to extract data from the HTML source. There are several libraries available to
 achieve this, such as:
 
- * `BeautifulSoup`_ is a very popular web scraping library among Python
-   programmers which constructs a Python object based on the structure of the
-   HTML code and also deals with bad markup reasonably well, but it has one
-   drawback: it's slow.
+-   `BeautifulSoup`_ is a very popular web scraping library among Python
+    programmers which constructs a Python object based on the structure of the
+    HTML code and also deals with bad markup reasonably well, but it has one
+    drawback: it's slow.
 
- * `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
-   API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard
-   library.)
+-   `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
+    API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python
+    standard library.)
 
 Scrapy comes with its own mechanism for extracting data. They're called
 selectors because they "select" certain parts of the HTML document specified
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 0f46f1c87..b910fc453 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -95,20 +95,21 @@ convenience.
 Available Shortcuts
 -------------------
 
- * ``shelp()`` - print a help with the list of available objects and shortcuts
+-   ``shelp()`` - print a help with the list of available objects and
+    shortcuts
 
- * ``fetch(url[, redirect=True])`` - fetch a new response from the given
-   URL and update all related objects accordingly. You can optionaly ask for
-   HTTP 3xx redirections to not be followed by passing ``redirect=False``
+-   ``fetch(url[, redirect=True])`` - fetch a new response from the given URL
+    and update all related objects accordingly. You can optionaly ask for HTTP
+    3xx redirections to not be followed by passing ``redirect=False``
 
- * ``fetch(request)`` - fetch a new response from the given request and
-   update all related objects accordingly.
+-   ``fetch(request)`` - fetch a new response from the given request and update
+    all related objects accordingly.
 
- * ``view(response)`` - open the given response in your local web browser, for
-   inspection. This will add a `\<base\> tag`_ to the response body in order
-   for external links (such as images and style sheets) to display properly.
-   Note, however, that this will create a temporary file in your computer,
-   which won't be removed automatically.
+-   ``view(response)`` - open the given response in your local web browser, for
+    inspection. This will add a `\<base\> tag`_ to the response body in order
+    for external links (such as images and style sheets) to display properly.
+    Note, however, that this will create a temporary file in your computer,
+    which won't be removed automatically.
 
 .. _<base> tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base
 
@@ -122,21 +123,21 @@ content).
 
 Those objects are:
 
- * ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
+-    ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
 
- * ``spider`` - the Spider which is known to handle the URL, or a
-   :class:`~scrapy.spiders.Spider` object if there is no spider found for
-   the current URL
+-   ``spider`` - the Spider which is known to handle the URL, or a
+    :class:`~scrapy.spiders.Spider` object if there is no spider found for the
+    current URL
 
- * ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
-   page. You can modify this request using :meth:`~scrapy.http.Request.replace`
-   or fetch a new request (without leaving the shell) using the ``fetch``
-   shortcut.
+-   ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
+    page. You can modify this request using
+    :meth:`~scrapy.http.Request.replace` or fetch a new request (without
+    leaving the shell) using the ``fetch`` shortcut.
 
- * ``response`` - a :class:`~scrapy.http.Response` object containing the last
-   fetched page
+-   ``response`` - a :class:`~scrapy.http.Response` object containing the last
+    fetched page
 
- * ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
+-   ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
 
 Example of shell session
 ========================

From 4c801551fa10e4ff75f0768b903664f415a6a504 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 3 Feb 2021 21:11:46 +0100
Subject: [PATCH 257/716] Document that the bytes_received signal is not yet
 implemented for HTTP/2

---
 docs/topics/settings.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 05cb4a855..0a4684a91 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -703,6 +703,8 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
 
     -   No support for `server pushes`_, which are ignored.
 
+    -   No support for the :signal:`bytes_received` signal.
+
 .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2
 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
 .. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2

From 248800328cb32f79a214289d59e927b576e21712 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 3 Feb 2021 21:13:43 +0100
Subject: [PATCH 258/716] Fix test_pinned_twisted_version

---
 tests/test_dependencies.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index 93e7311d2..5e63ebffb 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -36,7 +36,7 @@ class ScrapyUtilsTest(unittest.TestCase):
         )
         config_parser = ConfigParser()
         config_parser.read(tox_config_file_path)
-        pattern = r'Twisted==([\d.]+)'
+        pattern = r'Twisted\[http2\]==([\d.]+)'
         match = re.search(pattern, config_parser['pinned']['deps'])
         pinned_twisted_version_string = match[1]
 

From 0e4b291701baa8b74bd13bd47831dd040dd7173f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 3 Feb 2021 21:28:04 +0100
Subject: [PATCH 259/716] HTTP/2: fix canceling a request before a connection
 has been established

---
 scrapy/core/http2/stream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index e345ca79a..572dbf7aa 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -459,7 +459,7 @@ class Stream:
             request=self._request,
             certificate=self._protocol.metadata['certificate'],
             ip_address=self._protocol.metadata['ip_address'],
-            protocol=to_unicode(self._protocol.transport.negotiatedProtocol),
+            protocol='h2',
         )
 
         self._deferred_response.callback(response)

From 7b11b74c77d1535f943baebbf5c794a63d147a13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 4 Feb 2021 11:08:01 +0100
Subject: [PATCH 260/716] Use --use-deprecated=legacy-resolver
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Let’s see how test results change
---
 tox.ini | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tox.ini b/tox.ini
index d8e900e06..ecd3aad6e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -27,6 +27,10 @@ passenv =
     GCS_PROJECT_ID
 #allow tox virtualenv to upgrade pip/wheel/setuptools
 download = true
+# TODO: Remove the custom install_command below
+# Temporary workaround to filter out errors caused by the insanely long time
+# that it takes for the new resolver to install dependencies.
+install_command=python -m pip install --use-deprecated=legacy-resolver {opts} {packages}
 commands =
     py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
 

From 76abcedaf4f31a7c76de9e19680dd7499d9eccf4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 13:36:25 +0500
Subject: [PATCH 261/716] Add as_async_generator.

---
 scrapy/utils/asyncgen.py     | 12 ++++++++++++
 tests/test_utils_asyncgen.py | 20 ++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 tests/test_utils_asyncgen.py

diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 7f697af5f..db2173f85 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,5 +1,17 @@
+import collections
+
+
 async def collect_asyncgen(result):
     results = []
     async for x in result:
         results.append(x)
     return results
+
+
+async def as_async_generator(it):
+    if isinstance(it, collections.abc.AsyncIterator):
+        async for r in it:
+            yield r
+    else:
+        for r in it:
+            yield r
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
new file mode 100644
index 000000000..9ae66c57c
--- /dev/null
+++ b/tests/test_utils_asyncgen.py
@@ -0,0 +1,20 @@
+from twisted.trial import unittest
+
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.defer import deferred_f_from_coro_f
+
+
+class AsyncgenUtilsTest(unittest.TestCase):
+    @deferred_f_from_coro_f
+    async def test_as_async_generator(self):
+        ag = as_async_generator(range(42))
+        results = []
+        async for i in ag:
+            results.append(i)
+        self.assertEqual(results, list(range(42)))
+
+    @deferred_f_from_coro_f
+    async def test_collect_asyncgen(self):
+        ag = as_async_generator(range(42))
+        results = await collect_asyncgen(ag)
+        self.assertEqual(results, list(range(42)))

From acff1eb4960940eb360f3cf00d5b33ed71acc351 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 14:36:38 +0500
Subject: [PATCH 262/716] Add aiter_errback.

---
 scrapy/utils/defer.py     | 14 ++++++++++++++
 tests/test_utils_defer.py | 31 +++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 6db9cc117..2d02c0621 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -124,6 +124,20 @@ def iter_errback(iterable, errback, *a, **kw):
             errback(failure.Failure(), *a, **kw)
 
 
+async def aiter_errback(aiterable, errback, *a, **kw):
+    """Wraps an async iterable calling an errback if an error is caught while
+    iterating it. Similar to scrapy.utils.defer.iter_errback()
+    """
+    it = aiterable.__aiter__()
+    while True:
+        try:
+            yield await it.__anext__()
+        except StopAsyncIteration:
+            break
+        except Exception:
+            errback(failure.Failure(), *a, **kw)
+
+
 def deferred_from_coro(o):
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, defer.Deferred):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index e60242a3b..06d91c574 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -2,12 +2,15 @@ from twisted.trial import unittest
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
+from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.defer import (
     iter_errback,
+    aiter_errback,
     mustbe_deferred,
     process_chain,
     process_chain_both,
     process_parallel,
+    deferred_f_from_coro_f,
 )
 
 
@@ -117,3 +120,31 @@ class IterErrbackTest(unittest.TestCase):
         self.assertEqual(out, [0, 1, 2, 3, 4])
         self.assertEqual(len(errors), 1)
         self.assertIsInstance(errors[0].value, ZeroDivisionError)
+
+
+class AiterErrbackTest(unittest.TestCase):
+
+    @deferred_f_from_coro_f
+    async def test_aiter_errback_good(self):
+        async def itergood():
+            for x in range(10):
+                yield x
+
+        errors = []
+        out = await collect_asyncgen(aiter_errback(itergood(), errors.append))
+        self.assertEqual(out, list(range(10)))
+        self.assertFalse(errors)
+
+    @deferred_f_from_coro_f
+    async def test_iter_errback_bad(self):
+        async def iterbad():
+            for x in range(10):
+                if x == 5:
+                    1 / 0
+                yield x
+
+        errors = []
+        out = await collect_asyncgen(aiter_errback(iterbad(), errors.append))
+        self.assertEqual(out, [0, 1, 2, 3, 4])
+        self.assertEqual(len(errors), 1)
+        self.assertIsInstance(errors[0].value, ZeroDivisionError)

From 7e9f498e00fd36c76ac139eb285526c9e70b4054 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 14:37:27 +0500
Subject: [PATCH 263/716] Add MutableAsyncChain.

---
 scrapy/utils/python.py     | 25 ++++++++++++++++
 tests/test_utils_python.py | 58 ++++++++++++++++++++++++++++++++++++--
 2 files changed, 81 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 5703fd4c3..0bf9bff70 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -12,6 +12,7 @@ from functools import partial, wraps
 from itertools import chain
 
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.asyncgen import as_async_generator
 from scrapy.utils.decorators import deprecated
 
 
@@ -355,3 +356,27 @@ class MutableChain:
     @deprecated("scrapy.utils.python.MutableChain.__next__")
     def next(self):
         return self.__next__()
+
+
+async def _async_chain(*iterables):
+    for it in iterables:
+        async for o in as_async_generator(it):
+            yield o
+
+
+class MutableAsyncChain:
+    """
+    Similar to MutableChain but for async iterables
+    """
+
+    def __init__(self, *args):
+        self.data = _async_chain(*args)
+
+    def extend(self, *aiterables):
+        self.data = _async_chain(self.data, _async_chain(*aiterables))
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        return await self.data.__anext__()
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3115cc92f..58b384591 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -2,15 +2,18 @@ import functools
 import gc
 import operator
 import platform
-import unittest
 from datetime import datetime
 from itertools import count
 from warnings import catch_warnings
 
+from twisted.trial import unittest
+
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
     WeakKeyCache, get_func_args, to_bytes, to_unicode,
-    without_none_values, MutableChain)
+    without_none_values, MutableChain, MutableAsyncChain)
 
 
 __doctests__ = ['scrapy.utils.python']
@@ -32,6 +35,57 @@ class MutableChainTest(unittest.TestCase):
         self.assertEqual(list(m), list(range(3, 13)))
 
 
+class MutableAsyncChainTest(unittest.TestCase):
+    @staticmethod
+    async def g1():
+        for i in range(3):
+            yield i
+
+    @staticmethod
+    async def g2():
+        return
+        yield
+
+    @staticmethod
+    async def g3():
+        for i in range(7, 10):
+            yield i
+
+    @staticmethod
+    async def g4():
+        for i in range(3, 5):
+            yield i
+        1 / 0
+        for i in range(5, 7):
+            yield i
+
+    @staticmethod
+    async def collect_asyncgen_exc(asyncgen):
+        results = []
+        async for x in asyncgen:
+            results.append(x)
+        return results
+
+    @deferred_f_from_coro_f
+    async def test_mutableasyncchain(self):
+        m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))
+        m.extend(self.g2())
+        m.extend(self.g3())
+
+        self.assertEqual(await m.__anext__(), 0)
+        results = await collect_asyncgen(m)
+        self.assertEqual(results, list(range(1, 10)))
+
+    @deferred_f_from_coro_f
+    async def test_mutableasyncchain_exc(self):
+        m = MutableAsyncChain(self.g1())
+        m.extend(self.g4())
+        m.extend(self.g3())
+
+        results = await collect_asyncgen(aiter_errback(m, lambda _: None))
+        self.assertEqual(results, list(range(5)))
+
+
 class ToUnicodeTest(unittest.TestCase):
     def test_converting_an_utf8_encoded_string_to_unicode(self):
         self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e')

From d658552f232547cc227e597b0d8489c4762eb9d2 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 15:20:04 +0500
Subject: [PATCH 264/716] Add only_not_asyncio.

---
 conftest.py | 6 ++++++
 pytest.ini  | 1 +
 2 files changed, 7 insertions(+)

diff --git a/conftest.py b/conftest.py
index 68b855c08..407bf9e62 100644
--- a/conftest.py
+++ b/conftest.py
@@ -55,5 +55,11 @@ def only_asyncio(request, reactor_pytest):
         pytest.skip('This test is only run with --reactor=asyncio')
 
 
+@pytest.fixture(autouse=True)
+def only_not_asyncio(request, reactor_pytest):
+    if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
+        pytest.skip('This test is only run without --reactor=asyncio')
+
+
 # Generate localhost certificate files, needed by some tests
 generate_keys()
diff --git a/pytest.ini b/pytest.ini
index d4deeb57c..416b228f9 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,6 +21,7 @@ addopts =
 twisted = 1
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
+    only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
 flake8-max-line-length = 119
 flake8-ignore =
     W503

From d66d52d3ed8aab0b4f126169c2855d00f4053907 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 15:21:12 +0500
Subject: [PATCH 265/716] Add process_iterable_helper.

---
 scrapy/utils/middlewares.py     | 35 +++++++++++++
 tests/test_utils_middlewares.py | 87 +++++++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+)
 create mode 100644 scrapy/utils/middlewares.py
 create mode 100644 tests/test_utils_middlewares.py

diff --git a/scrapy/utils/middlewares.py b/scrapy/utils/middlewares.py
new file mode 100644
index 000000000..da28e0ddf
--- /dev/null
+++ b/scrapy/utils/middlewares.py
@@ -0,0 +1,35 @@
+# coding: utf-8
+import inspect
+
+
+def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
+    for o in it:
+        if in_predicate and not in_predicate(o):
+            continue
+        if processor is not None:
+            o = processor(o)
+        if out_predicate and not out_predicate(o):
+            continue
+        yield o
+
+
+async def process_async_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
+    async for o in it:
+        if in_predicate and not in_predicate(o):
+            continue
+        if processor is not None:
+            o = processor(o)
+        if out_predicate and not out_predicate(o):
+            continue
+        yield o
+
+
+def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
+    """
+    For each item in the iterable: skips it if in_predicate is False, applies processor,
+    skips the result if out_predicate is False, else yields it.
+    """
+    if inspect.isasyncgen(it):
+        return process_async_iterable_helper(it, in_predicate, out_predicate, processor)
+    else:
+        return process_normal_iterable_helper(it, in_predicate, out_predicate, processor)
diff --git a/tests/test_utils_middlewares.py b/tests/test_utils_middlewares.py
new file mode 100644
index 000000000..d395ba1a9
--- /dev/null
+++ b/tests/test_utils_middlewares.py
@@ -0,0 +1,87 @@
+import collections
+
+from twisted.trial import unittest
+
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.defer import deferred_f_from_coro_f
+from scrapy.utils.middlewares import process_iterable_helper
+
+
+def predicate1(o):
+    return bool(o % 2)
+
+
+def predicate2(o):
+    return o < 10
+
+
+def processor(o):
+    return o * 2
+
+
+class ProcessIterableHelperNormalTest(unittest.TestCase):
+
+    def test_normal_in_predicate(self):
+        iterable1 = iter([1, 2, 3])
+        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1)
+        self.assertIsInstance(iterable2, collections.abc.Iterable)
+        list2 = list(iterable2)
+        self.assertEqual(list2, [1, 3])
+
+    def test_normal_out_predicate(self):
+        iterable1 = iter([1, 2, 10, 3, 15])
+        iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2)
+        self.assertIsInstance(iterable2, collections.abc.Iterable)
+        list2 = list(iterable2)
+        self.assertEqual(list2, [1, 2, 3])
+
+    def test_normal_processor(self):
+        iterable1 = iter([1, 2, 3])
+        iterable2 = process_iterable_helper(iterable1, processor=processor)
+        self.assertIsInstance(iterable2, collections.abc.Iterable)
+        list2 = list(iterable2)
+        self.assertEqual(list2, [2, 4, 6])
+
+    def test_normal_combined(self):
+        iterable1 = iter([1, 2, 10, 3, 6, 18, 5, 15])
+        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1,
+                                            out_predicate=predicate2, processor=processor)
+        self.assertIsInstance(iterable2, collections.abc.Iterable)
+        list2 = list(iterable2)
+        self.assertEqual(list2, [2, 6])
+
+
+class ProcessIterableHelperAsyncTest(unittest.TestCase):
+
+    @deferred_f_from_coro_f
+    async def test_async_in_predicate(self):
+        iterable1 = as_async_generator([1, 2, 3])
+        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1)
+        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
+        list2 = await collect_asyncgen(iterable2)
+        self.assertEqual(list2, [1, 3])
+
+    @deferred_f_from_coro_f
+    async def test_async_out_predicate(self):
+        iterable1 = as_async_generator([1, 2, 10, 3, 15])
+        iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2)
+        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
+        list2 = await collect_asyncgen(iterable2)
+        self.assertEqual(list2, [1, 2, 3])
+
+    @deferred_f_from_coro_f
+    async def test_async_processor(self):
+        iterable1 = as_async_generator([1, 2, 3])
+        iterable2 = process_iterable_helper(iterable1, processor=processor)
+        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
+        list2 = await collect_asyncgen(iterable2)
+        self.assertEqual(list2, [2, 4, 6])
+
+    @deferred_f_from_coro_f
+    async def test_async_combined(self):
+        iterable1 = as_async_generator([1, 2, 10, 3, 6, 18, 5, 15])
+        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1,
+                                            out_predicate=predicate2, processor=processor)
+        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
+        list2 = await collect_asyncgen(iterable2)
+        self.assertEqual(list2, [2, 6])

From 92f2c9e308a5eda361229a8e74f74a21b9ff770a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 15:22:20 +0500
Subject: [PATCH 266/716] Move spider middlewares to process_iterable_helper.

---
 scrapy/spidermiddlewares/depth.py     |  4 ++--
 scrapy/spidermiddlewares/offsite.py   | 32 ++++++++++++++-------------
 scrapy/spidermiddlewares/referer.py   |  3 ++-
 scrapy/spidermiddlewares/urllength.py |  3 ++-
 4 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 776a6879a..73079bca9 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -3,10 +3,10 @@ Depth Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
-
 import logging
 
 from scrapy.http import Request
+from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -55,4 +55,4 @@ class DepthMiddleware:
             if self.verbose_stats:
                 self.stats.inc_value('request_depth_count/0', spider=spider)
 
-        return (r for r in result or () if _filter(r))
+        return process_iterable_helper(result or (), in_predicate=_filter)
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 6e4efda97..e7f481269 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -10,6 +10,7 @@ import warnings
 from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -26,21 +27,22 @@ class OffsiteMiddleware:
         return o
 
     def process_spider_output(self, response, result, spider):
-        for x in result:
-            if isinstance(x, Request):
-                if x.dont_filter or self.should_follow(x, spider):
-                    yield x
-                else:
-                    domain = urlparse_cached(x).hostname
-                    if domain and domain not in self.domains_seen:
-                        self.domains_seen.add(domain)
-                        logger.debug(
-                            "Filtered offsite request to %(domain)r: %(request)s",
-                            {'domain': domain, 'request': x}, extra={'spider': spider})
-                        self.stats.inc_value('offsite/domains', spider=spider)
-                    self.stats.inc_value('offsite/filtered', spider=spider)
-            else:
-                yield x
+        def in_predicate(x):
+            if not isinstance(x, Request):
+                return True
+            if x.dont_filter or self.should_follow(x, spider):
+                return True
+            domain = urlparse_cached(x).hostname
+            if domain and domain not in self.domains_seen:
+                self.domains_seen.add(domain)
+                logger.debug(
+                    "Filtered offsite request to %(domain)r: %(request)s",
+                    {'domain': domain, 'request': x}, extra={'spider': spider})
+                self.stats.inc_value('offsite/domains', spider=spider)
+            self.stats.inc_value('offsite/filtered', spider=spider)
+            return False
+
+        return process_iterable_helper(result or (), in_predicate=in_predicate)
 
     def should_follow(self, request, spider):
         regex = self.host_regex
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index f81041376..91c8727e1 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -10,6 +10,7 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
+from scrapy.utils.middlewares import process_iterable_helper
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
@@ -337,7 +338,7 @@ class RefererMiddleware:
                 if referrer is not None:
                     r.headers.setdefault('Referer', referrer)
             return r
-        return (_set_referer(r) for r in result or ())
+        return process_iterable_helper(result or (), processor=_set_referer)
 
     def request_scheduled(self, request, spider):
         # check redirected request to patch "Referer" header if necessary
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index 5be1f80cb..c7359fecd 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -8,6 +8,7 @@ import logging
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
+from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -34,4 +35,4 @@ class UrlLengthMiddleware:
             else:
                 return True
 
-        return (r for r in result or () if _filter(r))
+        return process_iterable_helper(result or (), in_predicate=_filter)

From 2a1e9359caa0e16b336fdb2944ab7f4a69549896 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 5 Feb 2021 16:16:29 +0500
Subject: [PATCH 267/716] Add parallel_async.

---
 scrapy/core/scraper.py |  24 +++++++--
 scrapy/utils/defer.py  | 110 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 129 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 0d3e3450f..3eae71af7 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -1,6 +1,6 @@
 """This module implements the Scraper component which parses responses and
 extracts information from them"""
-
+import collections
 import logging
 from collections import deque
 
@@ -12,7 +12,16 @@ from scrapy import signals
 from scrapy.core.spidermw import SpiderMiddlewareManager
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel
+from scrapy.utils.defer import (
+    aiter_errback,
+    defer_fail,
+    defer_succeed,
+    deferred_from_coro,
+    iter_errback,
+    parallel,
+    parallel_async,
+)
+
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.spider import iterate_spider_output
@@ -180,9 +189,14 @@ class Scraper:
     def handle_spider_output(self, result, request, response, spider):
         if not result:
             return defer_succeed(None)
-        it = iter_errback(result, self.handle_spider_error, request, response, spider)
-        dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
-                       request, response, spider)
+        if isinstance(result, collections.abc.AsyncIterable):
+            it = aiter_errback(result, self.handle_spider_error, request, response, spider)
+            dfd = deferred_from_coro(parallel_async(it, self.concurrent_items, self._process_spidermw_output,
+                                     request, response, spider))
+        else:
+            it = iter_errback(result, self.handle_spider_error, request, response, spider)
+            dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
+                           request, response, spider)
         return dfd
 
     def _process_spidermw_output(self, output, request, response, spider):
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 2d02c0621..bd5f9c8fc 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -75,6 +75,116 @@ def parallel(iterable, count, callable, *args, **named):
     return defer.DeferredList([coop.coiterate(work) for _ in range(count)])
 
 
+class _AsyncCooperatorAdapter:
+    """ A class that wraps an async iterator into a normal iterator suitable
+    for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
+    it calls the callable directly in the callback, instead of providing a more
+    generic interface.
+
+    On the outside, this class behaves as an iterator that yields Deferreds.
+    Each Deferred is fired with the result of the callable which was called on
+    the next result from aiterator. It raises StopIteration when aiterator is
+    exhausted, as expected.
+
+    Cooperator calls __next__() multiple times and waits on the Deferreds
+    returned from it. As async generators (since Python 3.8) don't support
+    awaiting on __anext__() several times in parallel, we need to serialize
+    this. It's done by storing the Deferreds returned from __next__() and
+    firing the oldest one when a result from __anext__() is available.
+
+    The workflow:
+    1. When __next__() is called for the first time, it creates a Deferred, stores it
+    in self.waiting_deferreds and returns it. It also makes a Deferred that will wait
+    for self.aiterator.__anext__() and puts it into self.anext_deferred.
+    2. If __next__() is called again before self.anext_deferred fires, more Deferreds
+    are added to self.waiting_deferreds.
+    3. When self.anext_deferred fires, it either calls _callback() or _errback(). Both
+    clear self.anext_deferred.
+    3.1. _callback() calls the callable passing the result value that it takes, pops a
+    Deferred from self.waiting_deferreds, and if the callable result was a Deferred, it
+    chains those Deferreds so that the waiting Deferred will fire when the result
+    Deferred does, otherwise it fires it directly. This causes one awaiting task to
+    receive a result. If self.waiting_deferreds is still not empty, new __anext__() is
+    called and self.anext_deferred is populated.
+    3.2. _errback() checks the exception class. If it's StopAsyncIteration it means
+    self.aiterator is exhausted and so it sets self.finished and fires all
+    self.waiting_deferreds. Other exceptions are propagated.
+    4. If __next__() is called after __anext__() was handled, then if self.finished is
+    True, it raises StopIteration, otherwise it acts like in step 2, but if
+    self.anext_deferred is now empty is also populates it with a new __anext__().
+
+    Note that CooperativeTask ignores the value returned from the Deferred that it waits
+    for, so we fire them with None when needed.
+
+    It may be possible to write an async iterator-aware replacement for
+    Cooperator/CooperativeTask and use it instead of this adapter to achieve the same
+    goal.
+    """
+    def __init__(self, aiterator, callable, *callable_args, **callable_kwargs):
+        self.aiterator = aiterator
+        self.callable = callable
+        self.callable_args = callable_args
+        self.callable_kwargs = callable_kwargs
+        self.finished = False
+        self.waiting_deferreds = []
+        self.anext_deferred = None
+
+    def _callback(self, result):
+        # This gets called when the result from aiterator.__anext__() is available.
+        # It calls the callable on it and sends the result to the oldest waiting Deferred
+        # (by chaining if the result is a Deferred too or by firing if not).
+        self.anext_deferred = None
+        result = self.callable(result, *self.callable_args, **self.callable_kwargs)
+        d = self.waiting_deferreds.pop(0)
+        if d.called:
+            raise ValueError('Deferred in waiting_deferreds already called')
+        if isinstance(result, defer.Deferred):
+            result.chainDeferred(d)
+        else:
+            d.callback(None)
+        if self.waiting_deferreds:
+            self._call_anext()
+
+    def _errback(self, failure):
+        # This gets called on any exceptions in aiterator.__anext__().
+        # It handles StopAsyncIteration by stopping the iteration and reraises all others.
+        self.anext_deferred = None
+        failure.trap(StopAsyncIteration)
+        self.finished = True
+        for d in self.waiting_deferreds:
+            if d.called:
+                raise ValueError('Deferred in waiting_deferreds already called')
+            d.callback(None)
+
+    def _call_anext(self):
+        # This starts waiting for the next result from aiterator.
+        # If aiterator is exhausted, _errback will be called.
+        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
+        self.anext_deferred.addCallbacks(self._callback, self._errback)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        # This puts a new Deferred into self.waiting_deferreds and returns it.
+        # It also calls __anext__() if needed.
+        if self.finished:
+            raise StopIteration
+        d = defer.Deferred()
+        self.waiting_deferreds.append(d)
+        if not self.anext_deferred:
+            self._call_anext()
+        return d
+
+
+def parallel_async(async_iterable, count, callable, *args, **named):
+    """ Like parallel but for async iterables """
+    coop = task.Cooperator()
+    work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
+    dl = defer.DeferredList([coop.coiterate(work) for _ in range(count)])
+    return dl
+
+
 def process_chain(callbacks, input, *a, **kw):
     """Return a Deferred built by chaining the given callbacks"""
     d = defer.Deferred()

From 2152a2a50898b91a44e6b0c282b69b7bfb8d18f0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 5 Feb 2021 16:19:52 +0500
Subject: [PATCH 268/716] Add main infrastructure for async callbacks.

---
 scrapy/core/spidermw.py                     | 45 ++++++++++++++++-----
 scrapy/utils/defer.py                       | 19 +++++++++
 scrapy/utils/spider.py                      |  8 ++--
 scrapy/utils/test.py                        |  7 ++++
 tests/spiders.py                            | 38 ++++++++++++++++-
 tests/test_crawl.py                         | 33 +++++++++++++++
 tests/test_spidermiddleware_output_chain.py | 27 +++++++++++++
 7 files changed, 161 insertions(+), 16 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 763e0cdf6..961606f29 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,6 +3,7 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
+import inspect
 from itertools import islice
 
 from twisted.python.failure import Failure
@@ -11,11 +12,11 @@ from scrapy.exceptions import _InvalidOutput
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import mustbe_deferred
-from scrapy.utils.python import MutableChain
+from scrapy.utils.python import MutableAsyncChain, MutableChain
 
 
 def _isiterable(possible_iterator):
-    return hasattr(possible_iterator, '__iter__')
+    return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__')
 
 
 def _fname(f):
@@ -58,15 +59,31 @@ class SpiderMiddlewareManager(MiddlewareManager):
             return scrape_func(response, request, spider)
 
         def _evaluate_iterable(iterable, exception_processor_index, recover_to):
-            try:
-                for r in iterable:
-                    yield r
-            except Exception as ex:
+            def _process_exception(ex):
                 exception_result = process_spider_exception(Failure(ex), exception_processor_index)
                 if isinstance(exception_result, Failure):
                     raise
                 recover_to.extend(exception_result)
 
+            def _evaluate_normal_iterable(iterable):
+                try:
+                    for r in iterable:
+                        yield r
+                except Exception as ex:
+                    _process_exception(ex)
+
+            async def _evaluate_async_iterable(iterable):
+                try:
+                    async for r in iterable:
+                        yield r
+                except Exception as ex:
+                    _process_exception(ex)
+
+            if inspect.isasyncgen(iterable):
+                return _evaluate_async_iterable(iterable)
+            else:
+                return _evaluate_normal_iterable(iterable)
+
         def process_spider_exception(_failure, start_index=0):
             exception = _failure.value
             # don't handle _InvalidOutput exception
@@ -92,7 +109,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
         def process_spider_output(result, start_index=0):
             # items in this iterable do not need to go through the process_spider_output
             # chain, they went through it already from the process_spider_exception method
-            recovered = MutableChain()
+            if inspect.isasyncgen(result):
+                iter_class = MutableAsyncChain
+            else:
+                iter_class = MutableChain
+            recovered = iter_class()
 
             method_list = islice(self.methods['process_spider_output'], start_index, None)
             for method_index, method in enumerate(method_list, start=start_index):
@@ -113,12 +134,16 @@ class SpiderMiddlewareManager(MiddlewareManager):
                            f"iterable, got {type(result)}")
                     raise _InvalidOutput(msg)
 
-            return MutableChain(result, recovered)
+            return iter_class(result, recovered)
 
         def process_callback_output(result):
-            recovered = MutableChain()
+            if inspect.isasyncgen(result):
+                iter_class = MutableAsyncChain
+            else:
+                iter_class = MutableChain
+            recovered = iter_class()
             result = _evaluate_iterable(result, 0, recovered)
-            return MutableChain(process_spider_output(result), recovered)
+            return iter_class(process_spider_output(result), recovered)
 
         dfd = mustbe_deferred(process_spider_input, response)
         dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index bd5f9c8fc..554edc38c 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -290,3 +290,22 @@ def maybeDeferred_coro(f, *args, **kw):
         return defer.fail(result)
     else:
         return defer.succeed(result)
+
+
+def deferred_to_future(d):
+    """ Wraps a Deferred into a Future. Requires the asyncio reactor.
+    """
+    return d.asFuture(asyncio.get_event_loop())
+
+
+def maybe_deferred_to_future(d):
+    """ Converts a Deferred to something that can be awaited in a callback or other user coroutine.
+
+    If the asyncio reactor is installed, coroutines are wrapped into Futures, and only Futures can be
+    awaited inside them. Otherwise, coroutines are wrapped into Deferreds and Deferreds can be awaited
+    directly inside them.
+    """
+    if not is_asyncio_reactor_installed():
+        return d
+    else:
+        return deferred_to_future(d)
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 59fc9202f..d0fd1757d 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -4,7 +4,6 @@ import logging
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.asyncgen import collect_asyncgen
 
 
 logger = logging.getLogger(__name__)
@@ -12,14 +11,13 @@ logger = logging.getLogger(__name__)
 
 def iterate_spider_output(result):
     if inspect.isasyncgen(result):
-        d = deferred_from_coro(collect_asyncgen(result))
-        d.addCallback(iterate_spider_output)
-        return d
+        return result
     elif inspect.iscoroutine(result):
         d = deferred_from_coro(result)
         d.addCallback(iterate_spider_output)
         return d
-    return arg_to_iter(result)
+    else:
+        return arg_to_iter(deferred_from_coro(result))
 
 
 def iter_spider_classes(module):
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 24c38283a..d8fc25094 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -110,3 +110,10 @@ def mock_google_cloud_storage():
     bucket_mock.blob.return_value = blob_mock
 
     return (client_mock, bucket_mock, blob_mock)
+
+
+def get_web_client_agent_req(url):
+    from twisted.internet import reactor
+    from twisted.web.client import Agent  # imports twisted.internet.reactor
+    agent = Agent(reactor)
+    return agent.request(b'GET', url.encode('utf-8'))
diff --git a/tests/spiders.py b/tests/spiders.py
index 106392ea6..3e0ec001b 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -14,7 +14,8 @@ from scrapy.item import Item
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
 from scrapy.spiders.crawl import CrawlSpider, Rule
-from scrapy.utils.test import get_from_asyncio_queue
+from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
+from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req
 
 
 class MockServerSpider(Spider):
@@ -148,6 +149,41 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
         return reqs
 
 
+class AsyncDefAsyncioGenExcSpider(SimpleSpider):
+    name = 'asyncdef_asyncio_gen_exc'
+
+    async def parse(self, response):
+        for i in range(10):
+            await asyncio.sleep(0.1)
+            yield {'foo': i}
+            if i > 5:
+                raise ValueError("Stopping the processing")
+
+
+class AsyncDefDeferredDirectSpider(SimpleSpider):
+    name = 'asyncdef_deferred_direct'
+
+    async def parse(self, response):
+        resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200"))
+        yield {'code': resp.code}
+
+
+class AsyncDefDeferredWrappedSpider(SimpleSpider):
+    name = 'asyncdef_deferred_wrapped'
+
+    async def parse(self, response):
+        resp = await deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200")))
+        yield {'code': resp.code}
+
+
+class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
+    name = 'asyncdef_deferred_wrapped'
+
+    async def parse(self, response):
+        resp = await maybe_deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200")))
+        yield {'code': resp.code}
+
+
 class AsyncDefAsyncioGenSpider(SimpleSpider):
 
     name = 'asyncdef_asyncio_gen'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 1083c1678..cda52f0d4 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -20,12 +20,16 @@ from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
 from tests.spiders import (
     AsyncDefAsyncioGenComplexSpider,
+    AsyncDefAsyncioGenExcSpider,
     AsyncDefAsyncioGenLoopSpider,
     AsyncDefAsyncioGenSpider,
     AsyncDefAsyncioReqsReturnSpider,
     AsyncDefAsyncioReturnSingleElementSpider,
     AsyncDefAsyncioReturnSpider,
     AsyncDefAsyncioSpider,
+    AsyncDefDeferredDirectSpider,
+    AsyncDefDeferredMaybeWrappedSpider,
+    AsyncDefDeferredWrappedSpider,
     AsyncDefSpider,
     BrokenStartRequestsSpider,
     BytesReceivedCallbackSpider,
@@ -430,6 +434,18 @@ class CrawlSpiderTestCase(TestCase):
         for i in range(10):
             self.assertIn({'foo': i}, items)
 
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncgen_parse_exc(self):
+        log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider)
+        log = str(log)
+        self.assertIn("Spider error processing", log)
+        self.assertIn("ValueError", log)
+        itemcount = stats.get_value('item_scraped_count')
+        self.assertEqual(itemcount, 7)
+        for i in range(7):
+            self.assertIn({'foo': i}, items)
+
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_asyncgen_parse_complex(self):
@@ -449,6 +465,23 @@ class CrawlSpiderTestCase(TestCase):
         for req_id in range(3):
             self.assertIn(f"Got response 200, req_id {req_id}", str(log))
 
+    @mark.only_not_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_deferred_direct(self):
+        _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider)
+        self.assertEqual(items, [{'code': 200}])
+
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_deferred_wrapped(self):
+        log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider)
+        self.assertEqual(items, [{'code': 200}])
+
+    @defer.inlineCallbacks
+    def test_async_def_deferred_maybe_wrapped(self):
+        _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider)
+        self.assertEqual(items, [{'code': 200}])
+
     @defer.inlineCallbacks
     def test_response_ssl_certificate_none(self):
         crawler = self.runner.create_crawler(SingleRequestSpider)
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 029bf8bd6..4d1a7fcb0 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -43,6 +43,23 @@ class RecoverySpider(Spider):
             raise TabError()
 
 
+class RecoveryAsyncGenSpider(RecoverySpider):
+    name = 'RecoveryAsyncGenSpider'
+
+    async def parse(self, response):
+        for r in super().parse(response):
+            yield r
+
+
+class RecoveryMiddleware:
+    def process_spider_exception(self, response, exception, spider):
+        spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
+        return [
+            {'from': 'process_spider_exception'},
+            Request(response.url, meta={'dont_fail': True}, dont_filter=True),
+        ]
+
+
 # ================================================================================
 # (1) exceptions from a spider middleware's process_spider_input method
 class FailProcessSpiderInputMiddleware:
@@ -307,6 +324,16 @@ class TestSpiderMiddleware(TestCase):
         self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
         self.assertIn("'item_scraped_count': 3", str(log))
 
+    @defer.inlineCallbacks
+    def test_recovery_asyncgen(self):
+        """
+        Same as test_recovery but with an async callback.
+        """
+        log = yield self.crawl_log(RecoveryAsyncGenSpider)
+        self.assertIn("Middleware: TabError exception caught", str(log))
+        self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
+        self.assertIn("'item_scraped_count': 3", str(log))
+
     @defer.inlineCallbacks
     def test_process_spider_input_without_errback(self):
         """

From 58f848130145649a0191e98b182977278e2b9b6c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 5 Feb 2021 16:20:32 +0500
Subject: [PATCH 269/716] Update docs.

---
 docs/topics/asyncio.rst    | 24 ++++++++++++++++++++++++
 docs/topics/coroutines.rst | 12 +++---------
 2 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 91e1cca0d..4addaa178 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -39,4 +39,28 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the
 use it instead of the default asyncio event loop.
 
 
+.. _asyncio-await-dfd:
 
+Awaiting on Deferreds
+=====================
+
+When the asyncio reactor isn't installed, you can await on Deferreds in the
+coroutines directly. When it is installed, this is not possible anymore, due to
+specifics of the Scrapy coroutine integration (the coroutines are wrapped into
+asyncio Futures, not into Deferreds directly), and you need to wrap them into
+Futures. Scrapy provides two helpers for this:
+
+.. autofunction:: scrapy.utils.defer.deferred_to_future
+.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future
+
+If you want to write universal code that works on any reactors,
+you should use ``maybe_deferred_to_future`` on all Deferreds::
+
+    from scrapy.utils.defer import maybe_deferred_to_future
+
+    class MySpider(Spider):
+        # ...
+        async def parse_with_deferred(self, response):
+            additional_response = await maybe_deferred_to_future(treq.get('https://additional.url'))
+            additional_data = await maybe_deferred_to_future(treq.content(additional_response))
+            # ... use response and additional_data to yield items and requests
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 3b1549bd3..279632653 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -17,15 +17,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :class:`~scrapy.http.Request` callbacks.
 
-    .. note:: The callback output is not processed until the whole callback
-        finishes.
-
-        As a side effect, if the callback raises an exception, none of its
-        output is processed.
-
-        This is a known caveat of the current implementation that we aim to
-        address in a future version of Scrapy.
-
 -   The :meth:`process_item` method of
     :ref:`item pipelines <topics-item-pipeline>`.
 
@@ -92,6 +83,9 @@ This means you can use many useful Python libraries providing such code::
           :mod:`asyncio` loop and to use them you need to
           :doc:`enable asyncio support in Scrapy<asyncio>`.
 
+.. note:: If you want to ``await`` on Deferreds, you may need to
+          :ref:`wrap them<asyncio-await-dfd>`.
+
 Common use cases for asynchronous code include:
 
 * requesting data from websites, databases and other services (in callbacks,

From 7afcd634ab7d71a465cd9406091d371dca105789 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 5 Feb 2021 13:04:54 +0100
Subject: [PATCH 270/716] Remove unused import

---
 scrapy/core/http2/stream.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 572dbf7aa..8a1b3e470 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -15,7 +15,6 @@ from twisted.web.client import ResponseFailed
 from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
-from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
     from scrapy.core.http2.protocol import H2ClientProtocol

From 8527b53e14e729504f6e382dd78ef6b0457fc7e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 5 Feb 2021 13:06:27 +0100
Subject: [PATCH 271/716] Revert "Use --use-deprecated=legacy-resolver"

This reverts commit 7b11b74c77d1535f943baebbf5c794a63d147a13.
---
 tox.ini | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index ecd3aad6e..d8e900e06 100644
--- a/tox.ini
+++ b/tox.ini
@@ -27,10 +27,6 @@ passenv =
     GCS_PROJECT_ID
 #allow tox virtualenv to upgrade pip/wheel/setuptools
 download = true
-# TODO: Remove the custom install_command below
-# Temporary workaround to filter out errors caused by the insanely long time
-# that it takes for the new resolver to install dependencies.
-install_command=python -m pip install --use-deprecated=legacy-resolver {opts} {packages}
 commands =
     py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
 

From 5cf403295d44bd2531ee3fe2f07057cf155e9804 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 5 Feb 2021 19:40:14 +0500
Subject: [PATCH 272/716] Remove a duplicate definition.

---
 tests/test_spidermiddleware_output_chain.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 4d1a7fcb0..088c14ca8 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -51,15 +51,6 @@ class RecoveryAsyncGenSpider(RecoverySpider):
             yield r
 
 
-class RecoveryMiddleware:
-    def process_spider_exception(self, response, exception, spider):
-        spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
-        return [
-            {'from': 'process_spider_exception'},
-            Request(response.url, meta={'dont_fail': True}, dont_filter=True),
-        ]
-
-
 # ================================================================================
 # (1) exceptions from a spider middleware's process_spider_input method
 class FailProcessSpiderInputMiddleware:

From 1e9b52c3e0e4eb51aaad37e796b3ee810fd919a6 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 8 Feb 2021 22:02:03 +0500
Subject: [PATCH 273/716] Refactor SpiderMiddlewareManager.scrape_response.

---
 scrapy/core/spidermw.py | 148 +++++++++++++++++++++-------------------
 1 file changed, 77 insertions(+), 71 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 763e0cdf6..289292da7 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -41,86 +41,92 @@ class SpiderMiddlewareManager(MiddlewareManager):
         process_spider_exception = getattr(mw, 'process_spider_exception', None)
         self.methods['process_spider_exception'].appendleft(process_spider_exception)
 
-    def scrape_response(self, scrape_func, response, request, spider):
-
-        def process_spider_input(response):
-            for method in self.methods['process_spider_input']:
-                try:
-                    result = method(response=response, spider=spider)
-                    if result is not None:
-                        msg = (f"Middleware {_fname(method)} must return None "
-                               f"or raise an exception, got {type(result)}")
-                        raise _InvalidOutput(msg)
-                except _InvalidOutput:
-                    raise
-                except Exception:
-                    return scrape_func(Failure(), request, spider)
-            return scrape_func(response, request, spider)
-
-        def _evaluate_iterable(iterable, exception_processor_index, recover_to):
+    def _process_spider_input(self, scrape_func, response, request, spider):
+        for method in self.methods['process_spider_input']:
             try:
-                for r in iterable:
-                    yield r
+                result = method(response=response, spider=spider)
+                if result is not None:
+                    msg = (f"Middleware {_fname(method)} must return None "
+                           f"or raise an exception, got {type(result)}")
+                    raise _InvalidOutput(msg)
+            except _InvalidOutput:
+                raise
+            except Exception:
+                return scrape_func(Failure(), request, spider)
+        return scrape_func(response, request, spider)
+
+    def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to):
+        try:
+            for r in iterable:
+                yield r
+        except Exception as ex:
+            exception_result = self._process_spider_exception(response, spider, Failure(ex),
+                                                              exception_processor_index)
+            if isinstance(exception_result, Failure):
+                raise
+            recover_to.extend(exception_result)
+
+    def _process_spider_exception(self, response, spider, _failure, start_index=0):
+        exception = _failure.value
+        # don't handle _InvalidOutput exception
+        if isinstance(exception, _InvalidOutput):
+            return _failure
+        method_list = islice(self.methods['process_spider_exception'], start_index, None)
+        for method_index, method in enumerate(method_list, start=start_index):
+            if method is None:
+                continue
+            result = method(response=response, exception=exception, spider=spider)
+            if _isiterable(result):
+                # stop exception handling by handing control over to the
+                # process_spider_output chain if an iterable has been returned
+                return self._process_spider_output(response, spider, result, method_index + 1)
+            elif result is None:
+                continue
+            else:
+                msg = (f"Middleware {_fname(method)} must return None "
+                       f"or an iterable, got {type(result)}")
+                raise _InvalidOutput(msg)
+        return _failure
+
+    def _process_spider_output(self, response, spider, result, start_index=0):
+        # items in this iterable do not need to go through the process_spider_output
+        # chain, they went through it already from the process_spider_exception method
+        recovered = MutableChain()
+
+        method_list = islice(self.methods['process_spider_output'], start_index, None)
+        for method_index, method in enumerate(method_list, start=start_index):
+            if method is None:
+                continue
+            try:
+                # might fail directly if the output value is not a generator
+                result = method(response=response, result=result, spider=spider)
             except Exception as ex:
-                exception_result = process_spider_exception(Failure(ex), exception_processor_index)
+                exception_result = self._process_spider_exception(response, spider, Failure(ex), method_index + 1)
                 if isinstance(exception_result, Failure):
                     raise
-                recover_to.extend(exception_result)
+                return exception_result
+            if _isiterable(result):
+                result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
+            else:
+                msg = (f"Middleware {_fname(method)} must return an "
+                       f"iterable, got {type(result)}")
+                raise _InvalidOutput(msg)
 
-        def process_spider_exception(_failure, start_index=0):
-            exception = _failure.value
-            # don't handle _InvalidOutput exception
-            if isinstance(exception, _InvalidOutput):
-                return _failure
-            method_list = islice(self.methods['process_spider_exception'], start_index, None)
-            for method_index, method in enumerate(method_list, start=start_index):
-                if method is None:
-                    continue
-                result = method(response=response, exception=exception, spider=spider)
-                if _isiterable(result):
-                    # stop exception handling by handing control over to the
-                    # process_spider_output chain if an iterable has been returned
-                    return process_spider_output(result, method_index + 1)
-                elif result is None:
-                    continue
-                else:
-                    msg = (f"Middleware {_fname(method)} must return None "
-                           f"or an iterable, got {type(result)}")
-                    raise _InvalidOutput(msg)
-            return _failure
+        return MutableChain(result, recovered)
 
-        def process_spider_output(result, start_index=0):
-            # items in this iterable do not need to go through the process_spider_output
-            # chain, they went through it already from the process_spider_exception method
-            recovered = MutableChain()
-
-            method_list = islice(self.methods['process_spider_output'], start_index, None)
-            for method_index, method in enumerate(method_list, start=start_index):
-                if method is None:
-                    continue
-                try:
-                    # might fail directly if the output value is not a generator
-                    result = method(response=response, result=result, spider=spider)
-                except Exception as ex:
-                    exception_result = process_spider_exception(Failure(ex), method_index + 1)
-                    if isinstance(exception_result, Failure):
-                        raise
-                    return exception_result
-                if _isiterable(result):
-                    result = _evaluate_iterable(result, method_index + 1, recovered)
-                else:
-                    msg = (f"Middleware {_fname(method)} must return an "
-                           f"iterable, got {type(result)}")
-                    raise _InvalidOutput(msg)
-
-            return MutableChain(result, recovered)
+    def _process_callback_output(self, response, spider, result):
+        recovered = MutableChain()
+        result = self._evaluate_iterable(response, spider, result, 0, recovered)
+        return MutableChain(self._process_spider_output(response, spider, result), recovered)
 
+    def scrape_response(self, scrape_func, response, request, spider):
         def process_callback_output(result):
-            recovered = MutableChain()
-            result = _evaluate_iterable(result, 0, recovered)
-            return MutableChain(process_spider_output(result), recovered)
+            return self._process_callback_output(response, spider, result)
 
-        dfd = mustbe_deferred(process_spider_input, response)
+        def process_spider_exception(_failure):
+            return self._process_spider_exception(response, spider, _failure)
+
+        dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
         dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
         return dfd
 

From 45345ba6b508fae426223039e491dd721eaac9a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 8 Feb 2021 17:56:29 +0100
Subject: [PATCH 274/716] Use constraints.txt to limit pip resolver
 backtracking

---
 tests/constraints.txt | 9 ++++++++-
 tox.ini               | 1 -
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/constraints.txt b/tests/constraints.txt
index 5655ac2d3..3b30e6bb5 100644
--- a/tests/constraints.txt
+++ b/tests/constraints.txt
@@ -1 +1,8 @@
-Twisted!=18.4.0
\ No newline at end of file
+# Request the latest known version or newer of some dependencies to prevent the
+# pip dependency resolver from spending too much time backtracking.
+attrs>=20.2.0
+Pillow>=8.0.1
+pytest>=6.2.1
+pytest-twisted>=1.13.1
+sybil>=2.0.0
+Twisted>=19.10.0
diff --git a/tox.ini b/tox.ini
index d8e900e06..9908a4d51 100644
--- a/tox.ini
+++ b/tox.ini
@@ -67,7 +67,6 @@ commands =
 
 [pinned]
 deps =
-    -ctests/constraints.txt
     cryptography==2.0
     cssselect==0.9.1
     h2==3.2.0

From bb72bba1786bc6a725df24a01dffba2e6e2cb7c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 8 Feb 2021 21:51:57 +0100
Subject: [PATCH 275/716] tox: apply upper constraints to all non-pinned
 package installations

---
 tests/{constraints.txt => upper-constraints.txt} | 8 ++++++++
 tox.ini                                          | 9 ++++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)
 rename tests/{constraints.txt => upper-constraints.txt} (64%)

diff --git a/tests/constraints.txt b/tests/upper-constraints.txt
similarity index 64%
rename from tests/constraints.txt
rename to tests/upper-constraints.txt
index 3b30e6bb5..c8c57deea 100644
--- a/tests/constraints.txt
+++ b/tests/upper-constraints.txt
@@ -1,8 +1,16 @@
 # Request the latest known version or newer of some dependencies to prevent the
 # pip dependency resolver from spending too much time backtracking.
 attrs>=20.2.0
+Automat>=0.8.0
+itemadapter>=0.1.1
+itemloaders>=1.0.3
+lxml>=4.6.1
+parsel>=1.5.2
 Pillow>=8.0.1
+pyOpenSSL>=20.0.0
 pytest>=6.2.1
 pytest-twisted>=1.13.1
+service_identity>=17.0.0
+six>=1.14.0
 sybil>=2.0.0
 Twisted>=19.10.0
diff --git a/tox.ini b/tox.ini
index 9908a4d51..2dfe8987c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,7 +9,6 @@ minversion = 1.7.0
 
 [testenv]
 deps =
-    -ctests/constraints.txt
     -rtests/requirements-py3.txt
     # mitmproxy does not support PyPy
     # mitmproxy does not support Windows when running Python < 3.7
@@ -29,6 +28,8 @@ passenv =
 download = true
 commands =
     py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
+install_command =
+    pip install -U -ctests/upper-constraints.txt {opts} {packages}
 
 [testenv:typing]
 basepython = python3
@@ -90,12 +91,15 @@ deps =
     Pillow==4.0.0
 setenv =
     _SCRAPY_PINNED=true
+install_command =
+    pip install -U {opts} {packages}
 
 [testenv:pinned]
 deps =
     {[pinned]deps}
     lxml==3.5.0
     PyDispatcher==2.0.5
+install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}
 
@@ -107,6 +111,7 @@ deps =
     # not need to build lxml from sources in a CI Windows job:
     lxml==3.8.0
     PyDispatcher==2.0.5
+install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}
 
@@ -123,6 +128,7 @@ commands =
 [testenv:asyncio-pinned]
 deps = {[testenv:pinned]deps}
 commands = {[testenv:asyncio]commands}
+install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}
 
@@ -138,6 +144,7 @@ deps =
     lxml==4.0.0
     PyPyDispatcher==2.1.0
 commands = {[testenv:pypy3]commands}
+install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}
 

From 9ac5b1d021562a20b2a7d437f8bacb6754426811 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 8 Feb 2021 22:31:20 +0100
Subject: [PATCH 276/716] Adjust test constraints

---
 tests/upper-constraints.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/upper-constraints.txt b/tests/upper-constraints.txt
index c8c57deea..75f337856 100644
--- a/tests/upper-constraints.txt
+++ b/tests/upper-constraints.txt
@@ -2,12 +2,13 @@
 # pip dependency resolver from spending too much time backtracking.
 attrs>=20.2.0
 Automat>=0.8.0
+botocore>=1.20.3
 itemadapter>=0.1.1
 itemloaders>=1.0.3
 lxml>=4.6.1
 parsel>=1.5.2
 Pillow>=8.0.1
-pyOpenSSL>=20.0.0
+pyOpenSSL>=17.5  # mitmproxy 4.0.4
 pytest>=6.2.1
 pytest-twisted>=1.13.1
 service_identity>=17.0.0

From 15b501c0898150de686ee42dd4d78a411891e795 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 10 Feb 2021 18:10:57 +0100
Subject: [PATCH 277/716] Do not force string interpolation while logging

---
 scrapy/core/http2/protocol.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 36a51b898..968bfce63 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -346,7 +346,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             elif isinstance(event, SettingsAcknowledged):
                 self.settings_acknowledged(event)
             elif isinstance(event, UnknownFrameReceived):
-                logger.warning(f'Unknown frame received: {event.frame}')
+                logger.warning('Unknown frame received: %s', event.frame)
 
     # Event handler functions starts here
     def connection_terminated(self, event: ConnectionTerminated) -> None:

From 67cff0e8a949a83220ae6d7d2c2ca40e6c8a8207 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 10 Feb 2021 22:44:14 +0500
Subject: [PATCH 278/716] Silence pylint "naked raise" error.

---
 scrapy/core/spidermw.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 96392aae7..d3d7e5f8c 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -61,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             exception_result = self._process_spider_exception(response, spider, Failure(ex),
                                                               exception_processor_index)
             if isinstance(exception_result, Failure):
-                raise
+                raise  # pylint: disable=E0704
             recover_to.extend(exception_result)
 
         def _evaluate_normal_iterable(iterable):

From 54fd371481ef0ff80cc23bfab7e7f53126a31bc3 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 11 Feb 2021 14:24:11 +0500
Subject: [PATCH 279/716] Skip uvloop 0.15.0+ on py36.

---
 tests/requirements-py3.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index a86c4ae4f..21a554624 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -9,7 +9,8 @@ pytest-twisted >= 1.11
 pytest-xdist
 sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures
-uvloop; platform_system != "Windows"
+uvloop < 0.15.0; platform_system != "Windows" and python_version == '3.6'
+uvloop; platform_system != "Windows" and python_version > '3.6'
 
 # optional for shell wrapper tests
 bpython

From abbbfbbb38a87b9f7259fe47493d5810b2439963 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 12 Feb 2021 22:41:16 +0500
Subject: [PATCH 280/716] Add tests for deferred_f_from_coro_f.

---
 tests/test_utils_defer.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index e60242a3b..7a5f458c7 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -1,8 +1,10 @@
+from pytest import mark
 from twisted.trial import unittest
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
 from scrapy.utils.defer import (
+    deferred_f_from_coro_f,
     iter_errback,
     mustbe_deferred,
     process_chain,
@@ -117,3 +119,18 @@ class IterErrbackTest(unittest.TestCase):
         self.assertEqual(out, [0, 1, 2, 3, 4])
         self.assertEqual(len(errors), 1)
         self.assertIsInstance(errors[0].value, ZeroDivisionError)
+
+
+class AsyncDefTestsuiteTest(unittest.TestCase):
+    @deferred_f_from_coro_f
+    async def test_deferred_f_from_coro_f(self):
+        pass
+
+    @deferred_f_from_coro_f
+    async def test_deferred_f_from_coro_f_generator(self):
+        yield
+
+    @mark.xfail(reason="Checks that the test is actually executed", strict=True)
+    @deferred_f_from_coro_f
+    async def test_deferred_f_from_coro_f_xfail(self):
+        raise Exception("This is expected to be raised")

From e80f37bd3fa0f2262e898f712541bb7a12e89110 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 17 Feb 2021 16:34:29 -0300
Subject: [PATCH 281/716] Test http2 agent for unsupported scheme

---
 tests/test_downloader_handlers_http2.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index 8f7f7aee0..bee9ae75b 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -3,6 +3,7 @@ from unittest import mock
 from twisted.internet import defer, error, reactor
 from twisted.trial import unittest
 from twisted.web import server
+from twisted.web.error import SchemeNotSupported
 
 from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
 from scrapy.http import Request
@@ -48,6 +49,12 @@ class Https2TestCase(Https11TestCase):
             reactor.callLater(.1, d.callback, logger)
             yield d
 
+    @defer.inlineCallbacks
+    def test_unsupported_scheme(self):
+        request = Request("ftp://unsupported.scheme")
+        d = self.download_request(request, Spider("foo"))
+        yield self.assertFailure(d, SchemeNotSupported)
+
     def test_download_broken_content_cause_data_loss(self, url='broken'):
         raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
 

From 4418f78941fc9d89ada9ad0436ebd0ae4ece1017 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 17 Feb 2021 18:36:52 -0300
Subject: [PATCH 282/716] Simplify check for negotiated protocol

negotiatedProtocol's type is Optional[bytes]

See https://github.com/twisted/twisted/blob/twisted-20.3.0/src/twisted/protocols/tls.py#L563-L587
and https://www.pyopenssl.org/en/20.0.1/api/ssl.html#OpenSSL.SSL.Connection.get_alpn_proto_negotiated

Note that OpenSSL.SSL.Connection.get_next_proto_negotiated is deprecated:
https://www.pyopenssl.org/en/20.0.0/changelog.html#backward-incompatible-changes
---
 scrapy/core/http2/protocol.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 968bfce63..9d7da14c1 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -233,13 +233,11 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
     def handshakeCompleted(self) -> None:
         """We close the connection with InvalidNegotiatedProtocol exception
         when the connection was not made via h2 protocol"""
-        negotiated_protocol = self.transport.negotiatedProtocol
-        if isinstance(negotiated_protocol, bytes):
-            negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8')
-        if negotiated_protocol != 'h2':
+        protocol = self.transport.negotiatedProtocol
+        if protocol is not None and protocol != b"h2":
             # Here we have not initiated the connection yet
             # So, no need to send a GOAWAY frame to the remote
-            self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)])
+            self._lose_connection_with_error([InvalidNegotiatedProtocol(protocol.decode("utf-8"))])
 
     def _check_received_data(self, data: bytes) -> None:
         """Checks for edge cases where the connection to remote fails

From 49af7c4c8b71f8643fe13c20261f97323117f980 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 18 Feb 2021 17:10:30 +0500
Subject: [PATCH 283/716] Drop pytest-twisted, use Scrapy code to install the
 reactor.

---
 conftest.py                | 15 +++++++++++++++
 pytest.ini                 |  1 -
 tests/requirements-py3.txt |  4 +---
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/conftest.py b/conftest.py
index 68b855c08..e4dd80de0 100644
--- a/conftest.py
+++ b/conftest.py
@@ -2,6 +2,8 @@ from pathlib import Path
 
 import pytest
 
+from scrapy.utils.reactor import install_reactor
+
 from tests.keys import generate_keys
 
 
@@ -40,6 +42,14 @@ def pytest_collection_modifyitems(session, config, items):
         pass
 
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--reactor",
+        default="default",
+        choices=["default", "asyncio"],
+    )
+
+
 @pytest.fixture(scope='class')
 def reactor_pytest(request):
     if not request.cls:
@@ -55,5 +65,10 @@ def only_asyncio(request, reactor_pytest):
         pytest.skip('This test is only run with --reactor=asyncio')
 
 
+def pytest_configure(config):
+    if config.getoption("--reactor") == "asyncio":
+        install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+
+
 # Generate localhost certificate files, needed by some tests
 generate_keys()
diff --git a/pytest.ini b/pytest.ini
index d4deeb57c..0aae09ff5 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -18,7 +18,6 @@ addopts =
     --ignore=docs/topics/stats.rst
     --ignore=docs/topics/telnetconsole.rst
     --ignore=docs/utils
-twisted = 1
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
 flake8-max-line-length = 119
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 21a554624..bd72c8c46 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -2,10 +2,8 @@
 attrs
 dataclasses; python_version == '3.6'
 pyftpdlib
-# https://github.com/pytest-dev/pytest-twisted/issues/93
-pytest != 5.4, != 5.4.1
+pytest
 pytest-cov
-pytest-twisted >= 1.11
 pytest-xdist
 sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures

From 40eab1d473a78369702490a901b5d304a40baf3b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 18 Feb 2021 19:56:12 +0500
Subject: [PATCH 284/716] Drop a duplicate import.

---
 tests/test_utils_defer.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index d220f969b..62f6ff194 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -5,14 +5,13 @@ from twisted.python.failure import Failure
 
 from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.defer import (
+    aiter_errback,
     deferred_f_from_coro_f,
     iter_errback,
-    aiter_errback,
     mustbe_deferred,
     process_chain,
     process_chain_both,
     process_parallel,
-    deferred_f_from_coro_f,
 )
 
 

From f9a5385146c741825d8c0b089cf8e8318dca9403 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 19 Feb 2021 21:19:21 +0500
Subject: [PATCH 285/716] Revert "Move spider middlewares to
 process_iterable_helper."

This reverts commit 92f2c9e308a5eda361229a8e74f74a21b9ff770a.
---
 scrapy/spidermiddlewares/depth.py     |  4 ++--
 scrapy/spidermiddlewares/offsite.py   | 32 +++++++++++++--------------
 scrapy/spidermiddlewares/referer.py   |  3 +--
 scrapy/spidermiddlewares/urllength.py |  3 +--
 4 files changed, 19 insertions(+), 23 deletions(-)

diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 73079bca9..776a6879a 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -3,10 +3,10 @@ Depth Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
+
 import logging
 
 from scrapy.http import Request
-from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -55,4 +55,4 @@ class DepthMiddleware:
             if self.verbose_stats:
                 self.stats.inc_value('request_depth_count/0', spider=spider)
 
-        return process_iterable_helper(result or (), in_predicate=_filter)
+        return (r for r in result or () if _filter(r))
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index e7f481269..6e4efda97 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -10,7 +10,6 @@ import warnings
 from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -27,22 +26,21 @@ class OffsiteMiddleware:
         return o
 
     def process_spider_output(self, response, result, spider):
-        def in_predicate(x):
-            if not isinstance(x, Request):
-                return True
-            if x.dont_filter or self.should_follow(x, spider):
-                return True
-            domain = urlparse_cached(x).hostname
-            if domain and domain not in self.domains_seen:
-                self.domains_seen.add(domain)
-                logger.debug(
-                    "Filtered offsite request to %(domain)r: %(request)s",
-                    {'domain': domain, 'request': x}, extra={'spider': spider})
-                self.stats.inc_value('offsite/domains', spider=spider)
-            self.stats.inc_value('offsite/filtered', spider=spider)
-            return False
-
-        return process_iterable_helper(result or (), in_predicate=in_predicate)
+        for x in result:
+            if isinstance(x, Request):
+                if x.dont_filter or self.should_follow(x, spider):
+                    yield x
+                else:
+                    domain = urlparse_cached(x).hostname
+                    if domain and domain not in self.domains_seen:
+                        self.domains_seen.add(domain)
+                        logger.debug(
+                            "Filtered offsite request to %(domain)r: %(request)s",
+                            {'domain': domain, 'request': x}, extra={'spider': spider})
+                        self.stats.inc_value('offsite/domains', spider=spider)
+                    self.stats.inc_value('offsite/filtered', spider=spider)
+            else:
+                yield x
 
     def should_follow(self, request, spider):
         regex = self.host_regex
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 91c8727e1..f81041376 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -10,7 +10,6 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
-from scrapy.utils.middlewares import process_iterable_helper
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
@@ -338,7 +337,7 @@ class RefererMiddleware:
                 if referrer is not None:
                     r.headers.setdefault('Referer', referrer)
             return r
-        return process_iterable_helper(result or (), processor=_set_referer)
+        return (_set_referer(r) for r in result or ())
 
     def request_scheduled(self, request, spider):
         # check redirected request to patch "Referer" header if necessary
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index c7359fecd..5be1f80cb 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -8,7 +8,6 @@ import logging
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
-from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -35,4 +34,4 @@ class UrlLengthMiddleware:
             else:
                 return True
 
-        return process_iterable_helper(result or (), in_predicate=_filter)
+        return (r for r in result or () if _filter(r))

From 5fc27b1e6fc8532dc5bc08fc5abf5a0daa8ef0c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 22 Feb 2021 14:09:06 +0100
Subject: [PATCH 286/716] Remove RetrySpiderMixin and retry_request

---
 scrapy/downloadermiddlewares/retry.py | 55 ---------------------------
 1 file changed, 55 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 023ab7d60..5963dacdf 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -75,61 +75,6 @@ def get_retry_request(
         return None
 
 
-def retry_request(
-    request,
-    *,
-    reason,
-    spider,
-    max_retry_times=None,
-    priority_adjust=None,
-):
-    new_request = get_retry_request(
-        request,
-        reason=reason,
-        spider=spider,
-        max_retry_times=max_retry_times,
-        priority_adjust=priority_adjust,
-    )
-    if new_request:
-        return [new_request]
-    return []
-
-
-class RetrySpiderMixin:
-
-    def get_retry_request(
-        self,
-        request,
-        *,
-        reason,
-        max_retry_times=None,
-        priority_adjust=None,
-    ):
-        return get_retry_request(
-            request,
-            reason=reason,
-            spider=self,
-            max_retry_times=max_retry_times,
-            priority_adjust=priority_adjust,
-        )
-
-    def retry_request(
-        self,
-        request,
-        *,
-        reason,
-        max_retry_times=None,
-        priority_adjust=None,
-    ):
-        return retry_request(
-            request,
-            reason=reason,
-            spider=self,
-            max_retry_times=max_retry_times,
-            priority_adjust=priority_adjust,
-        )
-
-
 class RetryMiddleware:
 
     # IOError is raised by the HttpCompression middleware when trying to

From 825462615a8df8e2274cf71c8a9bbea68c89262a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 22 Feb 2021 14:09:48 +0100
Subject: [PATCH 287/716] =?UTF-8?q?get=5Fretry=5Frequest:=20set=20the=20de?=
 =?UTF-8?q?fault=20retry=20reason=20to=20=E2=80=9Cunspecified=E2=80=9D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/downloadermiddlewares/retry.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 5963dacdf..b8ead12ce 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -34,8 +34,8 @@ logger = logging.getLogger(__name__)
 def get_retry_request(
     request,
     *,
-    reason,
     spider,
+    reason='unspecified',
     max_retry_times=None,
     priority_adjust=None,
 ):

From ec836dcc9290f50bad27c874cd0c25a87781735c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 22 Feb 2021 14:15:28 +0100
Subject: [PATCH 288/716] Solve style issues

---
 scrapy/downloadermiddlewares/retry.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index b8ead12ce..5f9bc756c 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -69,9 +69,12 @@ def get_retry_request(
         return new_request
     else:
         stats.inc_value('retry/max_reached')
-        logger.error("Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s",
-                        {'request': request, 'retry_times': retry_times, 'reason': reason},
-                        extra={'spider': spider})
+        logger.error(
+            "Gave up retrying %(request)s (failed %(retry_times)d times): "
+            "%(reason)s",
+            {'request': request, 'retry_times': retry_times, 'reason': reason},
+            extra={'spider': spider},
+        )
         return None
 
 

From 6ab990181c6502624ceb0cca6783d99b30d30c20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 22 Feb 2021 14:48:03 +0100
Subject: [PATCH 289/716] Document get_retry_requests

---
 docs/topics/downloader-middleware.rst | 17 ++++++++++
 docs/topics/settings.rst              | 14 --------
 scrapy/downloadermiddlewares/retry.py | 49 +++++++++++++++++++++++----
 3 files changed, 59 insertions(+), 21 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 6801adc9c..b539c23df 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -892,6 +892,11 @@ settings (see the settings documentation for more info):
 If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_retry`` key
 set to True, the request will be ignored by this middleware.
 
+To retry requests from a spider callback, you can use the
+:func:`get_retry_request` function:
+
+.. autofunction:: get_retry_request
+
 RetryMiddleware Settings
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -932,6 +937,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
 it is a common code used to indicate server overload. It is not included by
 default because HTTP specs say so.
 
+.. setting:: RETRY_PRIORITY_ADJUST
+
+RETRY_PRIORITY_ADJUST
+---------------------
+
+Default: ``-1``
+
+Adjust retry request priority relative to original request:
+
+- a positive priority adjust means higher priority.
+- **a negative priority adjust (default) means lower priority.**
+
 
 .. _topics-dlmw-robots:
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0086a6c74..7c5e9ef6f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1188,20 +1188,6 @@ Adjust redirect request priority relative to original request:
 - **a positive priority adjust (default) means higher priority.**
 - a negative priority adjust means lower priority.
 
-.. setting:: RETRY_PRIORITY_ADJUST
-
-RETRY_PRIORITY_ADJUST
----------------------
-
-Default: ``-1``
-
-Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware``
-
-Adjust retry request priority relative to original request:
-
-- a positive priority adjust means higher priority.
-- **a negative priority adjust (default) means lower priority.**
-
 .. setting:: ROBOTSTXT_OBEY
 
 ROBOTSTXT_OBEY
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 5f9bc756c..046e3ea71 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -39,16 +39,51 @@ def get_retry_request(
     max_retry_times=None,
     priority_adjust=None,
 ):
+    """
+    Returns a new :class:`~scrapy.Request` object to retry the specified
+    request, or ``None`` if retries of the specified request have been
+    exhausted.
+
+    For example, in a :class:`~scrapy.Spider` callback, you could use it as
+    follows::
+
+        def parse(self, response):
+            if not response.text:
+                new_request = get_retry_request(
+                    response.request,
+                    spider=self,
+                    reason='empty',
+                )
+                if new_request:
+                    yield new_request
+                return
+
+    *spider* is the :class:`~scrapy.Spider` instance which is asking for the
+    retry request. It is used to access the :ref:`settings <topics-settings>`
+    and :ref:`stats <topics-stats>`, and to provide extra logging context (see
+    :func:`logging.debug`).
+
+    *reason* is a string or an :class:`Exception` object that indicates the
+    reason why the request needs to be retried. It is used to name retry stats.
+
+    *max_retry_times* is a number that determines the maximum number of times
+    that *request* can be retried. If not specified or ``None``, the number is
+    read from the :reqmeta:`max_retry_times` meta key of the request. If the
+    :reqmeta:`max_retry_times` meta key is not defined or ``None``, the number
+    is read from the :setting:`RETRY_TIMES` setting.
+
+    *priority_adjust* is a number that determines how the priority of the new
+    request changes in relation to *request*. If not specified, the number is
+    read from the :setting:`RETRY_PRIORITY_ADJUST` setting.
+    """
     settings = spider.crawler.settings
     stats = spider.crawler.stats
     retry_times = request.meta.get('retry_times', 0) + 1
-    request_max_retry_times = request.meta.get(
-        'max_retry_times',
-        max_retry_times,
-    )
-    if request_max_retry_times is None:
-        request_max_retry_times = settings.getint('RETRY_TIMES')
-    if retry_times <= request_max_retry_times:
+    if max_retry_times is None:
+        max_retry_times = request.meta.get('max_retry_times')
+    if max_retry_times is None:
+        max_retry_times = settings.getint('RETRY_TIMES')
+    if retry_times <= max_retry_times:
         logger.debug(
             "Retrying %(request)s (failed %(retry_times)d times): %(reason)s",
             {'request': request, 'retry_times': retry_times, 'reason': reason},

From 80f5003c88f8ee4c5529c1a6d7fc3981efef511d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 22 Feb 2021 16:38:38 +0100
Subject: [PATCH 290/716] Add tests for get_retry_request

---
 scrapy/downloadermiddlewares/retry.py    |  13 +-
 tests/test_downloadermiddleware_retry.py | 498 ++++++++++++++++++++---
 2 files changed, 459 insertions(+), 52 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 046e3ea71..0f24e5d28 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -10,6 +10,7 @@ Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
 import logging
+from inspect import isclass
 
 from twisted.internet import defer
 from twisted.internet.error import (
@@ -81,8 +82,8 @@ def get_retry_request(
     retry_times = request.meta.get('retry_times', 0) + 1
     if max_retry_times is None:
         max_retry_times = request.meta.get('max_retry_times')
-    if max_retry_times is None:
-        max_retry_times = settings.getint('RETRY_TIMES')
+        if max_retry_times is None:
+            max_retry_times = settings.getint('RETRY_TIMES')
     if retry_times <= max_retry_times:
         logger.debug(
             "Retrying %(request)s (failed %(retry_times)d times): %(reason)s",
@@ -96,6 +97,8 @@ def get_retry_request(
             priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
         new_request.priority = request.priority + priority_adjust
 
+        if isclass(reason):
+            reason = reason()
         if isinstance(reason, Exception):
             reason = global_object_name(reason.__class__)
 
@@ -149,10 +152,12 @@ class RetryMiddleware:
             return self._retry(request, exception, spider)
 
     def _retry(self, request, reason, spider):
+        max_retry_times = request.meta.get('max_retry_times', self.max_retry_times)
+        priority_adjust = request.meta.get('priority_adjust', self.priority_adjust)
         return get_retry_request(
             request,
             reason=reason,
             spider=spider,
-            max_retry_times=self.max_retry_times,
-            priority_adjust=self.priority_adjust,
+            max_retry_times=max_retry_times,
+            priority_adjust=priority_adjust,
         )
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 364ce0c89..cf01a7dff 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -4,18 +4,22 @@ from twisted.internet.error import (
     ConnectError,
     ConnectionDone,
     ConnectionLost,
-    ConnectionRefusedError,
     DNSLookupError,
     TCPTimedOutError,
-    TimeoutError,
 )
 from twisted.web.client import ResponseFailed
 
-from scrapy.downloadermiddlewares.retry import RetryMiddleware
-from scrapy.spiders import Spider
+from scrapy.downloadermiddlewares.retry import (
+    get_retry_request,
+    RetryMiddleware,
+)
+from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response
+from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
+from testfixtures import LogCapture
+
 
 class RetryTest(unittest.TestCase):
     def setUp(self):
@@ -119,82 +123,480 @@ class RetryTest(unittest.TestCase):
 
 
 class MaxRetryTimesTest(unittest.TestCase):
-    def setUp(self):
-        self.crawler = get_crawler(Spider)
-        self.spider = self.crawler._create_spider('foo')
-        self.mw = RetryMiddleware.from_crawler(self.crawler)
-        self.mw.max_retry_times = 2
-        self.invalid_url = 'http://www.scrapytest.org/invalid_url'
+
+    invalid_url = 'http://www.scrapytest.org/invalid_url'
+
+    def get_spider_and_middleware(self, settings=None):
+        crawler = get_crawler(Spider, settings or {})
+        spider = crawler._create_spider('foo')
+        middleware = RetryMiddleware.from_crawler(crawler)
+        return spider, middleware
 
     def test_with_settings_zero(self):
-
-        # SETTINGS: RETRY_TIMES = 0
-        self.mw.max_retry_times = 0
-
+        max_retry_times = 0
+        settings = {'RETRY_TIMES': max_retry_times}
+        spider, middleware = self.get_spider_and_middleware(settings)
         req = Request(self.invalid_url)
-        self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
+        self._test_retry(
+            req,
+            DNSLookupError('foo'),
+            max_retry_times,
+            spider=spider,
+            middleware=middleware,
+        )
 
     def test_with_metakey_zero(self):
-
-        # SETTINGS: meta(max_retry_times) = 0
-        meta_max_retry_times = 0
-
-        req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
-        self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times)
+        max_retry_times = 0
+        spider, middleware = self.get_spider_and_middleware()
+        meta = {'max_retry_times': max_retry_times}
+        req = Request(self.invalid_url, meta=meta)
+        self._test_retry(
+            req,
+            DNSLookupError('foo'),
+            max_retry_times,
+            spider=spider,
+            middleware=middleware,
+        )
 
     def test_without_metakey(self):
-
-        # SETTINGS: RETRY_TIMES is NON-ZERO
-        self.mw.max_retry_times = 5
-
+        max_retry_times = 5
+        settings = {'RETRY_TIMES': max_retry_times}
+        spider, middleware = self.get_spider_and_middleware(settings)
         req = Request(self.invalid_url)
-        self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
+        self._test_retry(
+            req,
+            DNSLookupError('foo'),
+            max_retry_times,
+            spider=spider,
+            middleware=middleware,
+        )
 
     def test_with_metakey_greater(self):
-
-        # SETINGS: RETRY_TIMES < meta(max_retry_times)
-        self.mw.max_retry_times = 2
         meta_max_retry_times = 3
+        middleware_max_retry_times = 2
 
         req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
         req2 = Request(self.invalid_url)
 
-        self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times)
-        self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
+        settings = {'RETRY_TIMES': middleware_max_retry_times}
+        spider, middleware = self.get_spider_and_middleware(settings)
+
+        self._test_retry(
+            req1,
+            DNSLookupError('foo'),
+            meta_max_retry_times,
+            spider=spider,
+            middleware=middleware,
+        )
+        self._test_retry(
+            req2,
+            DNSLookupError('foo'),
+            middleware_max_retry_times,
+            spider=spider,
+            middleware=middleware,
+        )
 
     def test_with_metakey_lesser(self):
-
-        # SETINGS: RETRY_TIMES > meta(max_retry_times)
-        self.mw.max_retry_times = 5
         meta_max_retry_times = 4
+        middleware_max_retry_times = 5
 
         req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
         req2 = Request(self.invalid_url)
 
-        self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times)
-        self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
+        settings = {'RETRY_TIMES': middleware_max_retry_times}
+        spider, middleware = self.get_spider_and_middleware(settings)
+
+        self._test_retry(
+            req1,
+            DNSLookupError('foo'),
+            meta_max_retry_times,
+            spider=spider,
+            middleware=middleware,
+        )
+        self._test_retry(
+            req2,
+            DNSLookupError('foo'),
+            middleware_max_retry_times,
+            spider=spider,
+            middleware=middleware,
+        )
 
     def test_with_dont_retry(self):
+        max_retry_times = 4
+        spider, middleware = self.get_spider_and_middleware()
+        meta = {
+            'max_retry_times': max_retry_times,
+            'dont_retry': True,
+        }
+        req = Request(self.invalid_url, meta=meta)
+        self._test_retry(
+            req,
+            DNSLookupError('foo'),
+            0,
+            spider=spider,
+            middleware=middleware,
+        )
 
-        # SETTINGS: meta(max_retry_times) = 4
-        meta_max_retry_times = 4
-
-        req = Request(self.invalid_url, meta={
-            'max_retry_times': meta_max_retry_times, 'dont_retry': True
-        })
-
-        self._test_retry(req, DNSLookupError('foo'), 0)
-
-    def _test_retry(self, req, exception, max_retry_times):
+    def _test_retry(
+        self,
+        req,
+        exception,
+        max_retry_times,
+        spider=None,
+        middleware=None,
+    ):
+        spider = spider or self.spider
+        middleware = middleware or self.mw
 
         for i in range(0, max_retry_times):
-            req = self.mw.process_exception(req, exception, self.spider)
+            req = middleware.process_exception(req, exception, spider)
             assert isinstance(req, Request)
 
         # discard it
-        req = self.mw.process_exception(req, exception, self.spider)
+        req = middleware.process_exception(req, exception, spider)
         self.assertEqual(req, None)
 
 
+class GetRetryRequestTest(unittest.TestCase):
+
+    def get_spider(self, settings=None):
+        crawler = get_crawler(Spider, settings or {})
+        return crawler._create_spider('foo')
+
+    def test_basic_usage(self):
+        request = Request('https://example.com')
+        spider = self.get_spider()
+        with LogCapture() as log:
+            new_request = get_retry_request(
+                request,
+                spider=spider,
+            )
+        self.assertIsInstance(new_request, Request)
+        self.assertNotEqual(new_request, request)
+        self.assertEqual(new_request.dont_filter, True)
+        expected_retry_times = 1
+        self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
+        self.assertEqual(new_request.priority, -1)
+        expected_reason = "unspecified"
+        for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
+            self.assertEqual(spider.crawler.stats.get_value(stat), 1)
+        log.check_present(
+            (
+                "scrapy.downloadermiddlewares.retry",
+                "DEBUG",
+                f"Retrying {request} (failed {expected_retry_times} times): "
+                f"{expected_reason}",
+            )
+        )
+
+    def test_max_retries_reached(self):
+        request = Request('https://example.com')
+        spider = self.get_spider()
+        max_retry_times = 0
+        with LogCapture() as log:
+            new_request = get_retry_request(
+                request,
+                spider=spider,
+                max_retry_times=max_retry_times,
+            )
+        self.assertEqual(new_request, None)
+        self.assertEqual(
+            spider.crawler.stats.get_value('retry/max_reached'),
+            1
+        )
+        failure_count = max_retry_times + 1
+        expected_reason = "unspecified"
+        log.check_present(
+            (
+                "scrapy.downloadermiddlewares.retry",
+                "ERROR",
+                f"Gave up retrying {request} (failed {failure_count} times): "
+                f"{expected_reason}",
+            )
+        )
+
+    def test_one_retry(self):
+        request = Request('https://example.com')
+        spider = self.get_spider()
+        with LogCapture() as log:
+            new_request = get_retry_request(
+                request,
+                spider=spider,
+                max_retry_times=1,
+            )
+        self.assertIsInstance(new_request, Request)
+        self.assertNotEqual(new_request, request)
+        self.assertEqual(new_request.dont_filter, True)
+        expected_retry_times = 1
+        self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
+        self.assertEqual(new_request.priority, -1)
+        expected_reason = "unspecified"
+        for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
+            self.assertEqual(spider.crawler.stats.get_value(stat), 1)
+        log.check_present(
+            (
+                "scrapy.downloadermiddlewares.retry",
+                "DEBUG",
+                f"Retrying {request} (failed {expected_retry_times} times): "
+                f"{expected_reason}",
+            )
+        )
+
+    def test_two_retries(self):
+        spider = self.get_spider()
+        request = Request('https://example.com')
+        new_request = request
+        max_retry_times = 2
+        for index in range(max_retry_times):
+            with LogCapture() as log:
+                new_request = get_retry_request(
+                    new_request,
+                    spider=spider,
+                    max_retry_times=max_retry_times,
+                )
+            self.assertIsInstance(new_request, Request)
+            self.assertNotEqual(new_request, request)
+            self.assertEqual(new_request.dont_filter, True)
+            expected_retry_times = index+1
+            self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
+            self.assertEqual(new_request.priority, -expected_retry_times)
+            expected_reason = "unspecified"
+            for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
+                value = spider.crawler.stats.get_value(stat)
+                self.assertEqual(value, expected_retry_times)
+            log.check_present(
+                (
+                    "scrapy.downloadermiddlewares.retry",
+                    "DEBUG",
+                    f"Retrying {request} (failed {expected_retry_times} times): "
+                    f"{expected_reason}",
+                )
+            )
+
+        with LogCapture() as log:
+            new_request = get_retry_request(
+                new_request,
+                spider=spider,
+                max_retry_times=max_retry_times,
+            )
+        self.assertEqual(new_request, None)
+        self.assertEqual(
+            spider.crawler.stats.get_value('retry/max_reached'),
+            1
+        )
+        failure_count = max_retry_times + 1
+        expected_reason = "unspecified"
+        log.check_present(
+            (
+                "scrapy.downloadermiddlewares.retry",
+                "ERROR",
+                f"Gave up retrying {request} (failed {failure_count} times): "
+                f"{expected_reason}",
+            )
+        )
+
+    def test_no_spider(self):
+        request = Request('https://example.com')
+        with self.assertRaises(TypeError):
+            get_retry_request(request)
+
+    def test_max_retry_times_setting(self):
+        max_retry_times = 0
+        spider = self.get_spider({'RETRY_TIMES': max_retry_times})
+        request = Request('https://example.com')
+        new_request = get_retry_request(
+            request,
+            spider=spider,
+        )
+        self.assertEqual(new_request, None)
+
+    def test_max_retry_times_meta(self):
+        max_retry_times = 0
+        spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1})
+        meta = {'max_retry_times': max_retry_times}
+        request = Request('https://example.com', meta=meta)
+        new_request = get_retry_request(
+            request,
+            spider=spider,
+        )
+        self.assertEqual(new_request, None)
+
+    def test_max_retry_times_argument(self):
+        max_retry_times = 0
+        spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1})
+        meta = {'max_retry_times': max_retry_times + 1}
+        request = Request('https://example.com', meta=meta)
+        new_request = get_retry_request(
+            request,
+            spider=spider,
+            max_retry_times=max_retry_times,
+        )
+        self.assertEqual(new_request, None)
+
+    def test_priority_adjust_setting(self):
+        priority_adjust = 1
+        spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust})
+        request = Request('https://example.com')
+        new_request = get_retry_request(
+            request,
+            spider=spider,
+        )
+        self.assertEqual(new_request.priority, priority_adjust)
+
+    def test_priority_adjust_argument(self):
+        priority_adjust = 1
+        spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust+1})
+        request = Request('https://example.com')
+        new_request = get_retry_request(
+            request,
+            spider=spider,
+            priority_adjust=priority_adjust,
+        )
+        self.assertEqual(new_request.priority, priority_adjust)
+
+    def test_log_extra_retry_success(self):
+        request = Request('https://example.com')
+        spider = self.get_spider()
+        with LogCapture(attributes=('spider',)) as log:
+            new_request = get_retry_request(
+                request,
+                spider=spider,
+            )
+        log.check_present(spider)
+
+    def test_log_extra_retries_exceeded(self):
+        request = Request('https://example.com')
+        spider = self.get_spider()
+        with LogCapture(attributes=('spider',)) as log:
+            new_request = get_retry_request(
+                request,
+                spider=spider,
+                max_retry_times=0,
+            )
+        log.check_present(spider)
+
+    def test_reason_string(self):
+        request = Request('https://example.com')
+        spider = self.get_spider()
+        expected_reason = 'because'
+        with LogCapture() as log:
+            new_request = get_retry_request(
+                request,
+                spider=spider,
+                reason=expected_reason,
+            )
+        expected_retry_times = 1
+        for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
+            self.assertEqual(spider.crawler.stats.get_value(stat), 1)
+        log.check_present(
+            (
+                "scrapy.downloadermiddlewares.retry",
+                "DEBUG",
+                f"Retrying {request} (failed {expected_retry_times} times): "
+                f"{expected_reason}",
+            )
+        )
+
+    def test_reason_builtin_exception(self):
+        request = Request('https://example.com')
+        spider = self.get_spider()
+        expected_reason = NotImplementedError()
+        expected_reason_string = 'builtins.NotImplementedError'
+        with LogCapture() as log:
+            new_request = get_retry_request(
+                request,
+                spider=spider,
+                reason=expected_reason,
+            )
+        expected_retry_times = 1
+        stat = spider.crawler.stats.get_value(
+            f'retry/reason_count/{expected_reason_string}'
+        )
+        self.assertEqual(stat, 1)
+        log.check_present(
+            (
+                "scrapy.downloadermiddlewares.retry",
+                "DEBUG",
+                f"Retrying {request} (failed {expected_retry_times} times): "
+                f"{expected_reason}",
+            )
+        )
+
+    def test_reason_builtin_exception_class(self):
+        request = Request('https://example.com')
+        spider = self.get_spider()
+        expected_reason = NotImplementedError
+        expected_reason_string = 'builtins.NotImplementedError'
+        with LogCapture() as log:
+            new_request = get_retry_request(
+                request,
+                spider=spider,
+                reason=expected_reason,
+            )
+        expected_retry_times = 1
+        stat = spider.crawler.stats.get_value(
+            f'retry/reason_count/{expected_reason_string}'
+        )
+        self.assertEqual(stat, 1)
+        log.check_present(
+            (
+                "scrapy.downloadermiddlewares.retry",
+                "DEBUG",
+                f"Retrying {request} (failed {expected_retry_times} times): "
+                f"{expected_reason}",
+            )
+        )
+
+    def test_reason_custom_exception(self):
+        request = Request('https://example.com')
+        spider = self.get_spider()
+        expected_reason = IgnoreRequest()
+        expected_reason_string = 'scrapy.exceptions.IgnoreRequest'
+        with LogCapture() as log:
+            new_request = get_retry_request(
+                request,
+                spider=spider,
+                reason=expected_reason,
+            )
+        expected_retry_times = 1
+        stat = spider.crawler.stats.get_value(
+            f'retry/reason_count/{expected_reason_string}'
+        )
+        self.assertEqual(stat, 1)
+        log.check_present(
+            (
+                "scrapy.downloadermiddlewares.retry",
+                "DEBUG",
+                f"Retrying {request} (failed {expected_retry_times} times): "
+                f"{expected_reason}",
+            )
+        )
+
+    def test_reason_custom_exception_class(self):
+        request = Request('https://example.com')
+        spider = self.get_spider()
+        expected_reason = IgnoreRequest
+        expected_reason_string = 'scrapy.exceptions.IgnoreRequest'
+        with LogCapture() as log:
+            new_request = get_retry_request(
+                request,
+                spider=spider,
+                reason=expected_reason,
+            )
+        expected_retry_times = 1
+        stat = spider.crawler.stats.get_value(
+            f'retry/reason_count/{expected_reason_string}'
+        )
+        self.assertEqual(stat, 1)
+        log.check_present(
+            (
+                "scrapy.downloadermiddlewares.retry",
+                "DEBUG",
+                f"Retrying {request} (failed {expected_retry_times} times): "
+                f"{expected_reason}",
+            )
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

From 722a33a2ac8ebc22bb7a7056598898dcb76e98a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 22 Feb 2021 16:42:38 +0100
Subject: [PATCH 291/716] Fix style issues

---
 tests/test_downloadermiddleware_retry.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index cf01a7dff..61c0aaf2f 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -357,7 +357,7 @@ class GetRetryRequestTest(unittest.TestCase):
             self.assertIsInstance(new_request, Request)
             self.assertNotEqual(new_request, request)
             self.assertEqual(new_request.dont_filter, True)
-            expected_retry_times = index+1
+            expected_retry_times = index + 1
             self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
             self.assertEqual(new_request.priority, -expected_retry_times)
             expected_reason = "unspecified"
@@ -445,7 +445,7 @@ class GetRetryRequestTest(unittest.TestCase):
 
     def test_priority_adjust_argument(self):
         priority_adjust = 1
-        spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust+1})
+        spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust + 1})
         request = Request('https://example.com')
         new_request = get_retry_request(
             request,
@@ -458,7 +458,7 @@ class GetRetryRequestTest(unittest.TestCase):
         request = Request('https://example.com')
         spider = self.get_spider()
         with LogCapture(attributes=('spider',)) as log:
-            new_request = get_retry_request(
+            get_retry_request(
                 request,
                 spider=spider,
             )
@@ -468,7 +468,7 @@ class GetRetryRequestTest(unittest.TestCase):
         request = Request('https://example.com')
         spider = self.get_spider()
         with LogCapture(attributes=('spider',)) as log:
-            new_request = get_retry_request(
+            get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=0,
@@ -480,7 +480,7 @@ class GetRetryRequestTest(unittest.TestCase):
         spider = self.get_spider()
         expected_reason = 'because'
         with LogCapture() as log:
-            new_request = get_retry_request(
+            get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
@@ -503,7 +503,7 @@ class GetRetryRequestTest(unittest.TestCase):
         expected_reason = NotImplementedError()
         expected_reason_string = 'builtins.NotImplementedError'
         with LogCapture() as log:
-            new_request = get_retry_request(
+            get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
@@ -528,7 +528,7 @@ class GetRetryRequestTest(unittest.TestCase):
         expected_reason = NotImplementedError
         expected_reason_string = 'builtins.NotImplementedError'
         with LogCapture() as log:
-            new_request = get_retry_request(
+            get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
@@ -553,7 +553,7 @@ class GetRetryRequestTest(unittest.TestCase):
         expected_reason = IgnoreRequest()
         expected_reason_string = 'scrapy.exceptions.IgnoreRequest'
         with LogCapture() as log:
-            new_request = get_retry_request(
+            get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
@@ -578,7 +578,7 @@ class GetRetryRequestTest(unittest.TestCase):
         expected_reason = IgnoreRequest
         expected_reason_string = 'scrapy.exceptions.IgnoreRequest'
         with LogCapture() as log:
-            new_request = get_retry_request(
+            get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,

From 1f7665c4cfb955ca4e81d7bdb249cd88ada788c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 22 Feb 2021 16:48:10 +0100
Subject: [PATCH 292/716] Silence a PyLint check on a mistake made for testing
 purposes

---
 tests/test_downloadermiddleware_retry.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 61c0aaf2f..46e525f99 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -398,7 +398,7 @@ class GetRetryRequestTest(unittest.TestCase):
     def test_no_spider(self):
         request = Request('https://example.com')
         with self.assertRaises(TypeError):
-            get_retry_request(request)
+            get_retry_request(request)  # pylint: disable=missing-kwoa
 
     def test_max_retry_times_setting(self):
         max_retry_times = 0

From 6326178bc5824e8c08d84b6543de6977a16fb8d2 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Mon, 22 Feb 2021 12:50:51 -0300
Subject: [PATCH 293/716] http2: acceptable protocol update, tests (#4994)

---
 scrapy/core/http2/protocol.py       | 26 ++++++++++++++------------
 tests/test_http2_client_protocol.py |  8 ++++++++
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 9d7da14c1..6ca69b23b 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -33,13 +33,16 @@ from scrapy.spiders import Spider
 logger = logging.getLogger(__name__)
 
 
+PROTOCOL_NAME = b"h2"
+
+
 class InvalidNegotiatedProtocol(H2Error):
 
-    def __init__(self, negotiated_protocol: str) -> None:
+    def __init__(self, negotiated_protocol: bytes) -> None:
         self.negotiated_protocol = negotiated_protocol
 
     def __str__(self) -> str:
-        return f'InvalidNegotiatedProtocol: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}'
+        return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}")
 
 
 class RemoteTerminatedConnection(H2Error):
@@ -52,7 +55,7 @@ class RemoteTerminatedConnection(H2Error):
         self.terminate_event = event
 
     def __str__(self) -> str:
-        return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address!r}'
+        return f'Received GOAWAY frame from {self.remote_ip_address!r}'
 
 
 class MethodNotAllowed405(H2Error):
@@ -60,7 +63,7 @@ class MethodNotAllowed405(H2Error):
         self.remote_ip_address = remote_ip_address
 
     def __str__(self) -> str:
-        return f"MethodNotAllowed405: Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}"
+        return f"Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}"
 
 
 @implementer(IHandshakeListener)
@@ -231,13 +234,12 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self.transport.loseConnection()
 
     def handshakeCompleted(self) -> None:
-        """We close the connection with InvalidNegotiatedProtocol exception
-        when the connection was not made via h2 protocol"""
-        protocol = self.transport.negotiatedProtocol
-        if protocol is not None and protocol != b"h2":
-            # Here we have not initiated the connection yet
-            # So, no need to send a GOAWAY frame to the remote
-            self._lose_connection_with_error([InvalidNegotiatedProtocol(protocol.decode("utf-8"))])
+        """
+        Close the connection if it's not made via the expected protocol
+        """
+        if self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME:
+            # we have not initiated the connection yet, no need to send a GOAWAY frame to the remote peer
+            self._lose_connection_with_error([InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)])
 
     def _check_received_data(self, data: bytes) -> None:
         """Checks for edge cases where the connection to remote fails
@@ -414,4 +416,4 @@ class H2ClientFactory(Factory):
         return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred)
 
     def acceptableProtocols(self) -> List[bytes]:
-        return [b'h2']
+        return [PROTOCOL_NAME]
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index d9ab553f0..8b2f6a11d 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -5,6 +5,7 @@ import re
 import shutil
 import string
 from ipaddress import IPv4Address
+from unittest import mock
 from urllib.parse import urlencode
 
 from h2.exceptions import InvalidBodyLengthError
@@ -381,6 +382,13 @@ class Https2ClientProtocolTestCase(TestCase):
             200
         )
 
+    @inlineCallbacks
+    def test_invalid_negotiated_protocol(self):
+        with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"):
+            request = Request(url=self.get_url('/status?n=200'))
+            with self.assertRaises(ResponseFailed):
+                yield self.make_request(request)
+
     def test_cancel_request(self):
         request = Request(url=self.get_url('/get-data-html-large'))
 

From 7605f19ec429fd3adb6a18da8e48143f99f6bfec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 23 Feb 2021 05:54:48 +0100
Subject: [PATCH 294/716] HTTP/2: test 2 concurrent requests to the same domain

---
 tests/test_downloader_handlers_http2.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index bee9ae75b..44d45b7d8 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -73,6 +73,21 @@ class Https2TestCase(Https11TestCase):
     def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
         raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
 
+    def test_concurrent_requests_same_domain(self):
+        spider = Spider('foo')
+
+        request1 = Request(self.getURL('file'))
+        d1 = self.download_request(request1, spider)
+        d1.addCallback(lambda r: r.body)
+        d1.addCallback(self.assertEqual, b"0123456789")
+
+        request2 = Request(self.getURL('echo'), method='POST')
+        d2 = self.download_request(request2, spider)
+        d2.addCallback(lambda r: r.headers['Content-Length'])
+        d2.addCallback(self.assertEqual, b"79")
+
+        return defer.DeferredList([d1, d2])
+
 
 class Https2WrongHostnameTestCase(Https2TestCase):
     tls_log_message = (

From bd29f32dee445c77e7f2427a3047b7c74505efb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 23 Feb 2021 06:42:28 +0100
Subject: [PATCH 295/716] HTTP/2: do not make conn_lost_deferred optional

---
 scrapy/core/http2/protocol.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 6ca69b23b..1d150b7ce 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -70,7 +70,7 @@ class MethodNotAllowed405(H2Error):
 class H2ClientProtocol(Protocol, TimeoutMixin):
     IDLE_TIMEOUT = 240
 
-    def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None:
+    def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None:
         """
         Arguments:
             uri -- URI of the base url to which HTTP/2 Connection will be made.
@@ -308,8 +308,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         if not reason.check(connectionDone):
             self._conn_lost_errors.append(reason)
 
-        if self._conn_lost_deferred:
-            self._conn_lost_deferred.callback(self._conn_lost_errors)
+        self._conn_lost_deferred.callback(self._conn_lost_errors)
 
         for stream in self.streams.values():
             if stream.metadata['request_sent']:
@@ -407,7 +406,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
 @implementer(IProtocolNegotiationFactory)
 class H2ClientFactory(Factory):
-    def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None:
+    def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None:
         self.uri = uri
         self.settings = settings
         self.conn_lost_deferred = conn_lost_deferred

From 5ba31cd1a268475db1f3cc64d4e6febb477e8f77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 23 Feb 2021 11:57:33 +0100
Subject: [PATCH 296/716] HTTP/2 stream close reason handling: Use else +
 assert instead of elif

---
 scrapy/core/http2/stream.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 8a1b3e470..aa44c08ce 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -431,7 +431,8 @@ class Stream:
             errors.insert(0, InactiveStreamClosed(self._request))
             self._deferred_response.errback(ResponseFailed(errors))
 
-        elif reason is StreamCloseReason.INVALID_HOSTNAME:
+        else:
+            assert reason is StreamCloseReason.INVALID_HOSTNAME
             self._deferred_response.errback(InvalidHostname(
                 self._request,
                 str(self._protocol.metadata['uri'].host, 'utf-8'),

From 510109420733e93db0d525302555224a8364ed5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 24 Feb 2021 07:33:39 +0100
Subject: [PATCH 297/716] HTTP/2: test a CONNECT request

---
 tests/test_downloader_handlers_http2.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index 44d45b7d8..b5a40468a 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -1,5 +1,6 @@
 from unittest import mock
 
+from pytest import mark
 from twisted.internet import defer, error, reactor
 from twisted.trial import unittest
 from twisted.web import server
@@ -88,6 +89,14 @@ class Https2TestCase(Https11TestCase):
 
         return defer.DeferredList([d1, d2])
 
+    @mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247")
+    def test_connect_request(self):
+        request = Request(self.getURL('file'), method='CONNECT')
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.body)
+        d.addCallback(self.assertEqual, b'')
+        return d
+
 
 class Https2WrongHostnameTestCase(Https2TestCase):
     tls_log_message = (

From a36f952198101bfdfeadf4bec079db120809dbb1 Mon Sep 17 00:00:00 2001
From: Wehzie <39304339+Wehzie@users.noreply.github.com>
Date: Wed, 24 Feb 2021 08:15:44 +0100
Subject: [PATCH 298/716] fixed typo "an quotes.json" -> "a quotes.json"
 (#5005)

---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 9270ff42c..740e47d0c 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -464,7 +464,7 @@ The simplest way to store the scraped data is by using :ref:`Feed exports
 
     scrapy crawl quotes -O quotes.json
 
-That will generate an ``quotes.json`` file containing all scraped items,
+That will generate a ``quotes.json`` file containing all scraped items,
 serialized in `JSON`_.
 
 The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead

From 12064d799b8f15eef770a615237932b880b594a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 24 Feb 2021 10:37:38 +0100
Subject: [PATCH 299/716] HTTP/2: improve header handling

---
 scrapy/core/http2/stream.py             | 21 ++++++++---
 tests/test_downloader_handlers_http2.py | 46 +++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index aa44c08ce..8a701e7c6 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -220,11 +220,24 @@ class Stream:
                 (':path', path),
             ]
 
-        for name, value in self._request.headers.items():
-            headers.append((str(name, 'utf-8'), str(value[0], 'utf-8')))
+        content_length = str(len(self._request.body))
+        headers.append(('Content-Length', content_length))
 
-        if b'Content-Length' not in self._request.headers.keys():
-            headers.append(('Content-Length', str(len(self._request.body))))
+        content_length_name = self._request.headers.normkey(b'Content-Length')
+        for name, values in self._request.headers.items():
+            for value in values:
+                value = str(value, 'utf-8')
+                if name == content_length_name:
+                    if value != content_length:
+                        logger.warning(
+                            'Ignoring bad Content-Length header %r of request %r, '
+                            'sending %r instead',
+                            value,
+                            self._request,
+                            content_length,
+                        )
+                    continue
+                headers.append((str(name, 'utf-8'), value))
 
         return headers
 
diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index b5a40468a..7c3db5835 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -1,6 +1,8 @@
+import json
 from unittest import mock
 
 from pytest import mark
+from testfixtures import LogCapture
 from twisted.internet import defer, error, reactor
 from twisted.trial import unittest
 from twisted.web import server
@@ -97,6 +99,50 @@ class Https2TestCase(Https11TestCase):
         d.addCallback(self.assertEqual, b'')
         return d
 
+    def test_custom_content_length_good(self):
+        request = Request(self.getURL('contentlength'))
+        custom_content_length = str(len(request.body))
+        request.headers['Content-Length'] = custom_content_length
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.text)
+        d.addCallback(self.assertEqual, custom_content_length)
+        return d
+
+    def test_custom_content_length_bad(self):
+        request = Request(self.getURL('contentlength'))
+        actual_content_length = str(len(request.body))
+        bad_content_length = str(len(request.body)+1)
+        request.headers['Content-Length'] = bad_content_length
+        log = LogCapture()
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.text)
+        d.addCallback(self.assertEqual, actual_content_length)
+        d.addCallback(
+            lambda _: log.check_present(
+                (
+                    'scrapy.core.http2.stream',
+                    'WARNING',
+                    f'Ignoring bad Content-Length header '
+                    f'{bad_content_length!r} of request {request}, sending '
+                    f'{actual_content_length!r} instead',
+                )
+            )
+        )
+        d.addCallback(
+            lambda _: log.uninstall()
+        )
+        return d
+
+    def test_duplicate_header(self):
+        request = Request(self.getURL('echo'))
+        header, value1, value2 = 'Custom-Header', 'foo', 'bar'
+        request.headers.appendlist(header, value1)
+        request.headers.appendlist(header, value2)
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: json.loads(r.text)['headers'][header])
+        d.addCallback(self.assertEqual, [value1, value2])
+        return d
+
 
 class Https2WrongHostnameTestCase(Https2TestCase):
     tls_log_message = (

From 386e2a51ae4ed7bd53374a5cadfdf380b58284ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 24 Feb 2021 10:41:01 +0100
Subject: [PATCH 300/716] tests/test_downloader_handlers_http2.py: fix style
 issue

---
 tests/test_downloader_handlers_http2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index 7c3db5835..439778014 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -111,7 +111,7 @@ class Https2TestCase(Https11TestCase):
     def test_custom_content_length_bad(self):
         request = Request(self.getURL('contentlength'))
         actual_content_length = str(len(request.body))
-        bad_content_length = str(len(request.body)+1)
+        bad_content_length = str(len(request.body) + 1)
         request.headers['Content-Length'] = bad_content_length
         log = LogCapture()
         d = self.download_request(request, Spider('foo'))

From 3894ebb1497b32959c405201f2e010292cf65098 Mon Sep 17 00:00:00 2001
From: Djiar <djiar@kth.se>
Date: Tue, 23 Feb 2021 15:34:53 +0100
Subject: [PATCH 301/716] Refactor curl_to_request_kwargs #5001 Co-authored-by:
 alkazaz alkazaz@kth.se Co-authored-by: swill swill@kth.se Co-authored-by:
 lerjevik lerjevik@kth.se Co-authored-by: aljica aljica@kth.se

---
 scrapy/utils/curl.py | 36 +++++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 6660b9dc0..d8b3deaa1 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -34,6 +34,26 @@ for argument in safe_to_ignore_arguments:
     curl_parser.add_argument(*argument, action='store_true')
 
 
+def _parse_headers_and_cookies(parsed_args):
+    headers = []
+    cookies = {}
+    for header in parsed_args.headers or ():
+        name, val = header.split(':', 1)
+        name = name.strip()
+        val = val.strip()
+        if name.title() == 'Cookie':
+            for name, morsel in SimpleCookie(val).items():
+                cookies[name] = morsel.value
+        else:
+            headers.append((name, val))
+
+    if parsed_args.auth:
+        user, password = parsed_args.auth.split(':', 1)
+        headers.append(('Authorization', basic_auth_header(user, password)))
+
+    return headers, cookies
+
+
 def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
     """Convert a cURL command syntax to Request kwargs.
 
@@ -70,21 +90,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
 
     result = {'method': method.upper(), 'url': url}
 
-    headers = []
-    cookies = {}
-    for header in parsed_args.headers or ():
-        name, val = header.split(':', 1)
-        name = name.strip()
-        val = val.strip()
-        if name.title() == 'Cookie':
-            for name, morsel in SimpleCookie(val).items():
-                cookies[name] = morsel.value
-        else:
-            headers.append((name, val))
-
-    if parsed_args.auth:
-        user, password = parsed_args.auth.split(':', 1)
-        headers.append(('Authorization', basic_auth_header(user, password)))
+    headers, cookies = _parse_headers_and_cookies(parsed_args)
 
     if headers:
         result['headers'] = headers

From f689615e8d61f1f651b869b7a6284ca6ca15cde7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 24 Feb 2021 12:54:56 +0100
Subject: [PATCH 302/716] Close files in the PerYearXmlExportPipeline
 documentation example

---
 docs/topics/exporters.rst | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 0a0a1765a..8648daded 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -50,18 +50,19 @@ value of one of their fields::
             self.year_to_exporter = {}
 
         def close_spider(self, spider):
-            for exporter in self.year_to_exporter.values():
+            for exporter, xml_file in self.year_to_exporter.values():
                 exporter.finish_exporting()
+                xml_file.close()
 
         def _exporter_for_item(self, item):
             adapter = ItemAdapter(item)
             year = adapter['year']
             if year not in self.year_to_exporter:
-                f = open(f'{year}.xml', 'wb')
-                exporter = XmlItemExporter(f)
+                xml_file = open(f'{year}.xml', 'wb')
+                exporter = XmlItemExporter(xml_file)
                 exporter.start_exporting()
-                self.year_to_exporter[year] = exporter
-            return self.year_to_exporter[year]
+                self.year_to_exporter[year] = (exporter, xml_file)
+            return self.year_to_exporter[year][0]
 
         def process_item(self, item, spider):
             exporter = self._exporter_for_item(item)

From 7a54580679f192c4f1b66775aaddf1bee1efe448 Mon Sep 17 00:00:00 2001
From: deepang17 <47976918+deepang17@users.noreply.github.com>
Date: Sun, 28 Feb 2021 15:33:09 +0530
Subject: [PATCH 303/716] DOCS:Cover scrapy-bench in the documentation

---
 docs/topics/benchmarking.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst
index b01a66188..4e53900ee 100644
--- a/docs/topics/benchmarking.rst
+++ b/docs/topics/benchmarking.rst
@@ -81,5 +81,4 @@ follow links, any custom spider you write will probably do more stuff which
 results in slower crawl rates. How slower depends on how much your spider does
 and how well it's written.
 
-In the future, more cases will be added to the benchmarking suite to cover
-other common scenarios.
+To use it as a project for more complex Scrapy benchmarking: https://github.com/scrapy/scrapy-bench

From b25616d107d88bb195f19dca4c7e886a9ba652d3 Mon Sep 17 00:00:00 2001
From: deepang17 <47976918+deepang17@users.noreply.github.com>
Date: Sun, 28 Feb 2021 16:26:46 +0530
Subject: [PATCH 304/716] DOCS: Cover scrapy-bench in the documentation

---
 docs/topics/benchmarking.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst
index 4e53900ee..b15836771 100644
--- a/docs/topics/benchmarking.rst
+++ b/docs/topics/benchmarking.rst
@@ -81,4 +81,5 @@ follow links, any custom spider you write will probably do more stuff which
 results in slower crawl rates. How slower depends on how much your spider does
 and how well it's written.
 
-To use it as a project for more complex Scrapy benchmarking: https://github.com/scrapy/scrapy-bench
+To use it as a project for more complex Scrapy benchmarking:
+https://github.com/scrapy/scrapy-bench

From 3c5668d0db4b11836bafcc91b12aba911e71f104 Mon Sep 17 00:00:00 2001
From: James McKinney <26463+jpmckinney@users.noreply.github.com>
Date: Mon, 1 Mar 2021 22:00:33 -0500
Subject: [PATCH 305/716] docs: Clarify there's one extension instance per
 spider

---
 docs/topics/extensions.rst | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 519f18b63..9e86fd0fe 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -7,8 +7,7 @@ Extensions
 The extensions framework provides a mechanism for inserting your own
 custom functionality into Scrapy.
 
-Extensions are just regular classes that are instantiated at Scrapy startup,
-when extensions are initialized.
+Extensions are just regular classes.
 
 Extension settings
 ==================
@@ -27,8 +26,8 @@ Loading & activating extensions
 ===============================
 
 Extensions are loaded and activated at startup by instantiating a single
-instance of the extension class. Therefore, all the extension initialization
-code must be performed in the class ``__init__`` method.
+instance of the extension class per spider being run. All the extension
+initialization code must be performed in the class ``__init__`` method.
 
 To make an extension available, add it to the :setting:`EXTENSIONS` setting in
 your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented

From 9e62355271fa39e67b06f00f5601cdb848c7894e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 2 Mar 2021 12:09:10 -0300
Subject: [PATCH 306/716] Allow logger/stats customization in get_retry_request

---
 scrapy/downloadermiddlewares/retry.py    | 16 ++++++---
 tests/test_downloadermiddleware_retry.py | 44 ++++++++++++++++++++----
 2 files changed, 50 insertions(+), 10 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 0f24e5d28..8955c7e4f 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -29,7 +29,8 @@ from scrapy.utils.response import response_status_message
 from scrapy.core.downloader.handlers.http11 import TunnelError
 from scrapy.utils.python import global_object_name
 
-logger = logging.getLogger(__name__)
+
+retry_logger = logging.getLogger(__name__)
 
 
 def get_retry_request(
@@ -39,6 +40,8 @@ def get_retry_request(
     reason='unspecified',
     max_retry_times=None,
     priority_adjust=None,
+    logger=retry_logger,
+    stats_base_key='retry',
 ):
     """
     Returns a new :class:`~scrapy.Request` object to retry the specified
@@ -76,6 +79,11 @@ def get_retry_request(
     *priority_adjust* is a number that determines how the priority of the new
     request changes in relation to *request*. If not specified, the number is
     read from the :setting:`RETRY_PRIORITY_ADJUST` setting.
+
+    *logger* is the logging.Logger object to be used when logging messages
+
+    *stats_base_key* is a string to be used as the base key for the
+    retry-related job stats
     """
     settings = spider.crawler.settings
     stats = spider.crawler.stats
@@ -102,11 +110,11 @@ def get_retry_request(
         if isinstance(reason, Exception):
             reason = global_object_name(reason.__class__)
 
-        stats.inc_value('retry/count')
-        stats.inc_value(f'retry/reason_count/{reason}')
+        stats.inc_value(f'{stats_base_key}/count')
+        stats.inc_value(f'{stats_base_key}/reason_count/{reason}')
         return new_request
     else:
-        stats.inc_value('retry/max_reached')
+        stats.inc_value(f'{stats_base_key}/max_reached')
         logger.error(
             "Gave up retrying %(request)s (failed %(retry_times)d times): "
             "%(reason)s",
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 46e525f99..915bd3a3e 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -1,4 +1,7 @@
+import logging
 import unittest
+
+from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.internet.error import (
     ConnectError,
@@ -9,17 +12,12 @@ from twisted.internet.error import (
 )
 from twisted.web.client import ResponseFailed
 
-from scrapy.downloadermiddlewares.retry import (
-    get_retry_request,
-    RetryMiddleware,
-)
+from scrapy.downloadermiddlewares.retry import get_retry_request, RetryMiddleware
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
-from testfixtures import LogCapture
-
 
 class RetryTest(unittest.TestCase):
     def setUp(self):
@@ -597,6 +595,40 @@ class GetRetryRequestTest(unittest.TestCase):
             )
         )
 
+    def test_custom_logger(self):
+        logger = logging.getLogger("custom-logger")
+        request = Request("https://example.com")
+        spider = self.get_spider()
+        expected_reason = "because"
+        with LogCapture() as log:
+            get_retry_request(
+                request,
+                spider=spider,
+                reason=expected_reason,
+                logger=logger,
+            )
+        log.check_present(
+            (
+                "custom-logger",
+                "DEBUG",
+                f"Retrying {request} (failed 1 times): {expected_reason}",
+            )
+        )
+
+    def test_custom_stats_key(self):
+        request = Request("https://example.com")
+        spider = self.get_spider()
+        expected_reason = "because"
+        stats_key = "custom_retry"
+        get_retry_request(
+            request,
+            spider=spider,
+            reason=expected_reason,
+            stats_base_key=stats_key,
+        )
+        for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"):
+            self.assertEqual(spider.crawler.stats.get_value(stat), 1)
+
 
 if __name__ == "__main__":
     unittest.main()

From 36f1dbf665abd8935c3adda9a5abfef959573cdb Mon Sep 17 00:00:00 2001
From: deepang17 <47976918+deepang17@users.noreply.github.com>
Date: Tue, 2 Mar 2021 22:12:44 +0530
Subject: [PATCH 307/716] DOCS: Covered scrapy-bench

---
 .vscode/settings.json        | 3 +++
 docs/topics/benchmarking.rst | 5 +++--
 2 files changed, 6 insertions(+), 2 deletions(-)
 create mode 100644 .vscode/settings.json

diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 000000000..500bc7007
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+    "python.linting.pylintEnabled": true
+}
\ No newline at end of file
diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst
index b15836771..3e671365b 100644
--- a/docs/topics/benchmarking.rst
+++ b/docs/topics/benchmarking.rst
@@ -81,5 +81,6 @@ follow links, any custom spider you write will probably do more stuff which
 results in slower crawl rates. How slower depends on how much your spider does
 and how well it's written.
 
-To use it as a project for more complex Scrapy benchmarking:
-https://github.com/scrapy/scrapy-bench
+.. _scrapy-bench: https://github.com/scrapy/scrapy-bench
+
+Use scrapy-bench_ for more complex benchmarking.
\ No newline at end of file

From 4fe26ae9701c95a05723a79649314da03e3ddc1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 2 Mar 2021 19:46:34 +0100
Subject: [PATCH 308/716] Limit tests to Twisted < 21

---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index e70aef2d2..69f52bd9f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -18,6 +18,8 @@ deps =
     # Extras
     botocore>=1.4.87
     Pillow>=4.0.0
+    # Twisted 21+ causes issues in tests that use skipIf
+    Twisted<21
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID

From 3d88ac605b04f4c60ca81ce509c0c3b25cfb8cd7 Mon Sep 17 00:00:00 2001
From: deepang17 <47976918+deepang17@users.noreply.github.com>
Date: Tue, 9 Mar 2021 17:19:34 +0530
Subject: [PATCH 309/716] FIX: Updated benchmarking.rst

---
 .vscode/settings.json        | 3 ---
 docs/topics/benchmarking.rst | 4 ++--
 2 files changed, 2 insertions(+), 5 deletions(-)
 delete mode 100644 .vscode/settings.json

diff --git a/.vscode/settings.json b/.vscode/settings.json
deleted file mode 100644
index 500bc7007..000000000
--- a/.vscode/settings.json
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-    "python.linting.pylintEnabled": true
-}
\ No newline at end of file
diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst
index 3e671365b..0643df6a6 100644
--- a/docs/topics/benchmarking.rst
+++ b/docs/topics/benchmarking.rst
@@ -81,6 +81,6 @@ follow links, any custom spider you write will probably do more stuff which
 results in slower crawl rates. How slower depends on how much your spider does
 and how well it's written.
 
-.. _scrapy-bench: https://github.com/scrapy/scrapy-bench
+Use scrapy-bench_ for more complex benchmarking.
 
-Use scrapy-bench_ for more complex benchmarking.
\ No newline at end of file
+.. _scrapy-bench: https://github.com/scrapy/scrapy-bench
\ No newline at end of file

From 3bea5e1a974b7cd99f75edb4ff728a9d7163a805 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 9 Mar 2021 16:19:51 +0100
Subject: [PATCH 310/716] Remove unused _is_data_lost method

---
 scrapy/core/http2/stream.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 8a701e7c6..c2a4b702f 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -362,14 +362,6 @@ class Stream:
         self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
         self.close(reason)
 
-    def _is_data_lost(self) -> bool:
-        assert self.metadata['stream_closed_server']
-
-        expected_size = self._response['flow_controlled_size']
-        received_body_size = int(self._response['headers'][b'Content-Length'])
-
-        return expected_size != received_body_size
-
     def close(
         self,
         reason: StreamCloseReason,

From 0c160882306a8e33f92815cc12dc8979f04f8f5e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Thu, 11 Mar 2021 11:52:35 -0300
Subject: [PATCH 311/716] headers_received signal (#4897)

---
 docs/faq.rst                               | 11 +--
 docs/topics/exceptions.rst                 |  7 +-
 docs/topics/request-response.rst           |  6 +-
 docs/topics/signals.rst                    | 32 +++++++-
 scrapy/core/downloader/handlers/http11.py  | 24 ++++++
 scrapy/signals.py                          |  1 +
 tests/spiders.py                           | 29 ++++++++
 tests/test_crawl.py                        | 26 ++++++-
 tests/test_engine.py                       | 85 ++++++----------------
 tests/test_engine_stop_download_bytes.py   | 60 +++++++++++++++
 tests/test_engine_stop_download_headers.py | 56 ++++++++++++++
 11 files changed, 260 insertions(+), 77 deletions(-)
 create mode 100644 tests/test_engine_stop_download_bytes.py
 create mode 100644 tests/test_engine_stop_download_headers.py

diff --git a/docs/faq.rst b/docs/faq.rst
index f492dfa30..9709885f6 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -398,11 +398,12 @@ How can I cancel the download of a given response?
 --------------------------------------------------
 
 In some situations, it might be useful to stop the download of a certain response.
-For instance, if you only need the first part of a large response and you would like
-to save resources by avoiding the download of the whole body.
-In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received`
-signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to
-the :ref:`topics-stop-response-download` topic for additional information and examples.
+For instance, sometimes you can determine whether or not you need the full contents
+of a response by inspecting its headers or the first bytes of its body. In that case,
+you could save resources by attaching a handler to the :class:`~scrapy.signals.bytes_received`
+or :class:`~scrapy.signals.headers_received` signals and raising a
+:exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the
+:ref:`topics-stop-response-download` topic for additional information and examples.
 
 
 .. _has been reported: https://github.com/scrapy/scrapy/issues/2905
diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 583a50ab8..2f1517906 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -85,8 +85,8 @@ StopDownload
 
 .. exception:: StopDownload(fail=True)
 
-Raised from a :class:`~scrapy.signals.bytes_received` signal handler to
-indicate that no further bytes should be downloaded for a response.
+Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
+signal handler to indicate that no further bytes should be downloaded for a response.
 
 The ``fail`` boolean parameter controls which method will handle the resulting
 response:
@@ -110,5 +110,6 @@ attribute.
     ``StopDownload(False)`` or ``StopDownload(True)`` will raise
     a :class:`TypeError`.
 
-See the documentation for the :class:`~scrapy.signals.bytes_received` signal
+See the documentation for the :class:`~scrapy.signals.bytes_received` and
+:class:`~scrapy.signals.headers_received` signals
 and the :ref:`topics-stop-response-download` topic for additional information and examples.
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 98906992d..37008f3e9 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -432,9 +432,9 @@ The meta key is used set retry times per request. When initialized, the
 Stopping the download of a Response
 ===================================
 
-Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a
-:class:`~scrapy.signals.bytes_received` signal handler will stop the
-download of a given response. See the following example::
+Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the
+:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
+signals will stop the download of a given response. See the following example::
 
     import scrapy
 
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 1d99d8c28..98cfa606c 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -384,6 +384,11 @@ bytes_received
     a possible scenario for a 25 kb response would be two signals fired
     with 10 kb of data, and a final one with 5 kb of data.
 
+    Handlers for this signal can stop the download of a response while it
+    is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
+    exception. Please refer to the :ref:`topics-stop-response-download` topic
+    for additional information and examples.
+
     This signal does not support returning deferreds from its handlers.
 
     :param data: the data received by the download handler
@@ -395,11 +400,36 @@ bytes_received
     :param spider: the spider associated with the response
     :type spider: :class:`~scrapy.spiders.Spider` object
 
-.. note:: Handlers of this signal can stop the download of a response while it
+headers_received
+~~~~~~~~~~~~~~~~
+
+.. versionadded:: VERSION
+
+.. signal:: headers_received
+.. function:: headers_received(headers, request, spider)
+
+    Sent by the HTTP 1.1 and S3 download handlers when the response headers are
+    available for a given request, before downloading any additional content.
+
+    Handlers for this signal can stop the download of a response while it
     is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
     exception. Please refer to the :ref:`topics-stop-response-download` topic
     for additional information and examples.
 
+    This signal does not support returning deferreds from its handlers.
+
+    :param headers: the headers received by the download handler
+    :type headers: :class:`scrapy.http.headers.Headers` object
+
+    :param body_length: expected size of the response body, in bytes
+    :type body_length: `int`
+
+    :param request: the request that generated the download
+    :type request: :class:`~scrapy.http.Request` object
+
+    :param spider: the spider associated with the response
+    :type spider: :class:`~scrapy.spiders.Spider` object
+
 Response signals
 ----------------
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 513df2de9..516a4326b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -382,6 +382,29 @@ class ScrapyAgent:
         return result
 
     def _cb_bodyready(self, txresponse, request):
+        headers_received_result = self._crawler.signals.send_catch_log(
+            signal=signals.headers_received,
+            headers=Headers(txresponse.headers.getAllRawHeaders()),
+            body_length=txresponse.length,
+            request=request,
+            spider=self._crawler.spider,
+        )
+        for handler, result in headers_received_result:
+            if isinstance(result, Failure) and isinstance(result.value, StopDownload):
+                logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
+                             {"request": request, "handler": handler.__qualname__})
+                txresponse._transport.stopProducing()
+                with suppress(AttributeError):
+                    txresponse._transport._producer.loseConnection()
+                return {
+                    "txresponse": txresponse,
+                    "body": b"",
+                    "flags": ["download_stopped"],
+                    "certificate": None,
+                    "ip_address": None,
+                    "failure": result if result.value.fail else None,
+                }
+
         # deliverBody hangs for responses without body
         if txresponse.length == 0:
             return {
@@ -529,6 +552,7 @@ class _ResponseReader(protocol.Protocol):
             if isinstance(result, Failure) and isinstance(result.value, StopDownload):
                 logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
                              {"request": self._request, "handler": handler.__qualname__})
+                self.transport.stopProducing()
                 self.transport._producer.loseConnection()
                 failure = result if result.value.fail else None
                 self._finish_response(flags=["download_stopped"], failure=failure)
diff --git a/scrapy/signals.py b/scrapy/signals.py
index c61ae6ec3..8cf2a4d93 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -17,6 +17,7 @@ request_reached_downloader = object()
 request_left_downloader = object()
 response_received = object()
 response_downloaded = object()
+headers_received = object()
 bytes_received = object()
 item_scraped = object()
 item_dropped = object()
diff --git a/tests/spiders.py b/tests/spiders.py
index 106392ea6..7e579098a 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -390,3 +390,32 @@ class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
     def bytes_received(self, data, request, spider):
         self.meta["bytes_received"] = data
         raise StopDownload(fail=True)
+
+
+class HeadersReceivedCallbackSpider(MetaSpider):
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = super().from_crawler(crawler, *args, **kwargs)
+        crawler.signals.connect(spider.headers_received, signals.headers_received)
+        return spider
+
+    def start_requests(self):
+        yield Request(self.mockserver.url("/status"), errback=self.errback)
+
+    def parse(self, response):
+        self.meta["response"] = response
+
+    def errback(self, failure):
+        self.meta["failure"] = failure
+
+    def headers_received(self, headers, body_length, request, spider):
+        self.meta["headers_received"] = headers
+        raise StopDownload(fail=False)
+
+
+class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):
+
+    def headers_received(self, headers, body_length, request, spider):
+        self.meta["headers_received"] = headers
+        raise StopDownload(fail=True)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 1083c1678..84bac9b50 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -35,6 +35,8 @@ from tests.spiders import (
     DelaySpider,
     DuplicateStartRequestsSpider,
     FollowAllSpider,
+    HeadersReceivedCallbackSpider,
+    HeadersReceivedErrbackSpider,
     SimpleSpider,
     SingleRequestSpider,
 )
@@ -496,7 +498,7 @@ class CrawlSpiderTestCase(TestCase):
         self.assertEqual(str(ip_address), gethostbyname(expected_netloc))
 
     @defer.inlineCallbacks
-    def test_stop_download_callback(self):
+    def test_bytes_received_stop_download_callback(self):
         crawler = self.runner.create_crawler(BytesReceivedCallbackSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("failure"))
@@ -505,7 +507,7 @@ class CrawlSpiderTestCase(TestCase):
         self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length)
 
     @defer.inlineCallbacks
-    def test_stop_download_errback(self):
+    def test_bytes_received_stop_download_errback(self):
         crawler = self.runner.create_crawler(BytesReceivedErrbackSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("response"))
@@ -518,3 +520,23 @@ class CrawlSpiderTestCase(TestCase):
         self.assertLess(
             len(crawler.spider.meta["failure"].value.response.body),
             crawler.spider.full_response_length)
+
+    @defer.inlineCallbacks
+    def test_headers_received_stop_download_callback(self):
+        crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertIsNone(crawler.spider.meta.get("failure"))
+        self.assertIsInstance(crawler.spider.meta["response"], Response)
+        self.assertEqual(crawler.spider.meta["response"].headers, crawler.spider.meta.get("headers_received"))
+
+    @defer.inlineCallbacks
+    def test_headers_received_stop_download_errback(self):
+        crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertIsNone(crawler.spider.meta.get("response"))
+        self.assertIsInstance(crawler.spider.meta["failure"], Failure)
+        self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)
+        self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response)
+        self.assertEqual(
+            crawler.spider.meta["failure"].value.response.headers,
+            crawler.spider.meta.get("headers_received"))
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 3629aa1aa..ef1204f94 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -19,14 +19,12 @@ from urllib.parse import urlparse
 import attr
 from itemadapter import ItemAdapter
 from pydispatch import dispatcher
-from testfixtures import LogCapture
 from twisted.internet import defer, reactor
 from twisted.trial import unittest
 from twisted.web import server, static, util
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine
-from scrapy.exceptions import StopDownload
 from scrapy.http import Request
 from scrapy.item import Item, Field
 from scrapy.linkextractors import LinkExtractor
@@ -143,6 +141,7 @@ class CrawlerRun:
         self.reqreached = []
         self.itemerror = []
         self.itemresp = []
+        self.headers = {}
         self.bytes = defaultdict(lambda: list())
         self.signals_caught = {}
         self.spider_class = spider_class
@@ -165,6 +164,7 @@ class CrawlerRun:
         self.crawler = get_crawler(self.spider_class)
         self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
         self.crawler.signals.connect(self.item_error, signals.item_error)
+        self.crawler.signals.connect(self.headers_received, signals.headers_received)
         self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
@@ -183,6 +183,7 @@ class CrawlerRun:
             if not name.startswith('_'):
                 disconnect_all(signal)
         self.deferred.callback(None)
+        return self.crawler.stop()
 
     def geturl(self, path):
         return f"http://localhost:{self.portno}{path}"
@@ -197,6 +198,9 @@ class CrawlerRun:
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
+    def headers_received(self, headers, body_length, request, spider):
+        self.headers[request] = headers
+
     def bytes_received(self, data, request, spider):
         self.bytes[request].append(data)
 
@@ -220,18 +224,7 @@ class CrawlerRun:
         self.signals_caught[sig] = signalargs
 
 
-class StopDownloadCrawlerRun(CrawlerRun):
-    """
-    Make sure raising the StopDownload exception stops the download of the response body
-    """
-
-    def bytes_received(self, data, request, spider):
-        super().bytes_received(data, request, spider)
-        raise StopDownload(fail=False)
-
-
 class EngineTest(unittest.TestCase):
-
     @defer.inlineCallbacks
     def test_crawler(self):
 
@@ -241,8 +234,8 @@ class EngineTest(unittest.TestCase):
             self.run = CrawlerRun(spider)
             yield self.run.run()
             self._assert_visited_urls()
-            self._assert_scheduled_requests(urls_to_visit=9)
-            self._assert_downloaded_responses()
+            self._assert_scheduled_requests(count=9)
+            self._assert_downloaded_responses(count=9)
             self._assert_scraped_items()
             self._assert_signals_caught()
             self._assert_bytes_received()
@@ -251,7 +244,7 @@ class EngineTest(unittest.TestCase):
     def test_crawler_dupefilter(self):
         self.run = CrawlerRun(TestDupeFilterSpider)
         yield self.run.run()
-        self._assert_scheduled_requests(urls_to_visit=8)
+        self._assert_scheduled_requests(count=8)
         self._assert_dropped_requests()
 
     @defer.inlineCallbacks
@@ -267,8 +260,8 @@ class EngineTest(unittest.TestCase):
         urls_expected = {self.run.geturl(p) for p in must_be_visited}
         assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"
 
-    def _assert_scheduled_requests(self, urls_to_visit=None):
-        self.assertEqual(urls_to_visit, len(self.run.reqplug))
+    def _assert_scheduled_requests(self, count=None):
+        self.assertEqual(count, len(self.run.reqplug))
 
         paths_expected = ['/item999.html', '/item2.html', '/item1.html']
 
@@ -286,10 +279,10 @@ class EngineTest(unittest.TestCase):
     def _assert_dropped_requests(self):
         self.assertEqual(len(self.run.reqdropped), 1)
 
-    def _assert_downloaded_responses(self):
+    def _assert_downloaded_responses(self, count):
         # response tests
-        self.assertEqual(9, len(self.run.respplug))
-        self.assertEqual(9, len(self.run.reqreached))
+        self.assertEqual(count, len(self.run.respplug))
+        self.assertEqual(count, len(self.run.reqreached))
 
         for response, _ in self.run.respplug:
             if self.run.getpath(response.url) == '/item999.html':
@@ -323,6 +316,13 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('Item 2 name', item['name'])
                 self.assertEqual('200', item['price'])
 
+    def _assert_headers_received(self):
+        for headers in self.run.headers.values():
+            self.assertIn(b"Server", headers)
+            self.assertIn(b"TwistedWeb", headers[b"Server"])
+            self.assertIn(b"Date", headers)
+            self.assertIn(b"Content-Type", headers)
+
     def _assert_bytes_received(self):
         self.assertEqual(9, len(self.run.bytes))
         for request, data in self.run.bytes.items():
@@ -371,6 +371,7 @@ class EngineTest(unittest.TestCase):
         assert signals.spider_opened in self.run.signals_caught
         assert signals.spider_idle in self.run.signals_caught
         assert signals.spider_closed in self.run.signals_caught
+        assert signals.headers_received in self.run.signals_caught
 
         self.assertEqual({'spider': self.run.spider},
                          self.run.signals_caught[signals.spider_opened])
@@ -403,48 +404,6 @@ class EngineTest(unittest.TestCase):
         self.assertEqual(len(e.open_spiders), 0)
 
 
-class StopDownloadEngineTest(EngineTest):
-
-    @defer.inlineCallbacks
-    def test_crawler(self):
-        for spider in TestSpider, DictItemsSpider:
-            self.run = StopDownloadCrawlerRun(spider)
-            with LogCapture() as log:
-                yield self.run.run()
-                log.check_present(("scrapy.core.downloader.handlers.http11",
-                                   "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> "
-                                   "from signal handler"
-                                   " StopDownloadCrawlerRun.bytes_received"))
-                log.check_present(("scrapy.core.downloader.handlers.http11",
-                                   "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/> "
-                                   "from signal handler"
-                                   " StopDownloadCrawlerRun.bytes_received"))
-                log.check_present(("scrapy.core.downloader.handlers.http11",
-                                   "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> "
-                                   "from signal handler"
-                                   " StopDownloadCrawlerRun.bytes_received"))
-            self._assert_visited_urls()
-            self._assert_scheduled_requests(urls_to_visit=9)
-            self._assert_downloaded_responses()
-            self._assert_signals_caught()
-            self._assert_bytes_received()
-
-    def _assert_bytes_received(self):
-        self.assertEqual(9, len(self.run.bytes))
-        for request, data in self.run.bytes.items():
-            joined_data = b"".join(data)
-            self.assertTrue(len(data) == 1)  # signal was fired only once
-            if self.run.getpath(request.url) == "/numbers":
-                # Received bytes are not the complete response. The exact amount depends
-                # on the buffer size, which can vary, so we only check that the amount
-                # of received bytes is strictly less than the full response.
-                numbers = [str(x).encode("utf8") for x in range(2**18)]
-                self.assertTrue(len(joined_data) < len(b"".join(numbers)))
-
-
 if __name__ == "__main__":
     if len(sys.argv) > 1 and sys.argv[1] == 'runserver':
         start_test_site(debug=True)
diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py
new file mode 100644
index 000000000..0ba69e096
--- /dev/null
+++ b/tests/test_engine_stop_download_bytes.py
@@ -0,0 +1,60 @@
+from testfixtures import LogCapture
+from twisted.internet import defer
+
+from scrapy.exceptions import StopDownload
+
+from tests.test_engine import (
+    AttrsItemsSpider,
+    DataClassItemsSpider,
+    DictItemsSpider,
+    TestSpider,
+    CrawlerRun,
+    EngineTest,
+)
+
+
+class BytesReceivedCrawlerRun(CrawlerRun):
+    def bytes_received(self, data, request, spider):
+        super().bytes_received(data, request, spider)
+        raise StopDownload(fail=False)
+
+
+class BytesReceivedEngineTest(EngineTest):
+    @defer.inlineCallbacks
+    def test_crawler(self):
+        for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
+            if spider is None:
+                continue
+            self.run = BytesReceivedCrawlerRun(spider)
+            with LogCapture() as log:
+                yield self.run.run()
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> "
+                                   "from signal handler BytesReceivedCrawlerRun.bytes_received"))
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   f"Download stopped for <GET http://localhost:{self.run.portno}/> "
+                                   "from signal handler BytesReceivedCrawlerRun.bytes_received"))
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> "
+                                   "from signal handler BytesReceivedCrawlerRun.bytes_received"))
+            self._assert_visited_urls()
+            self._assert_scheduled_requests(count=9)
+            self._assert_downloaded_responses(count=9)
+            self._assert_signals_caught()
+            self._assert_headers_received()
+            self._assert_bytes_received()
+
+    def _assert_bytes_received(self):
+        self.assertEqual(9, len(self.run.bytes))
+        for request, data in self.run.bytes.items():
+            joined_data = b"".join(data)
+            self.assertTrue(len(data) == 1)  # signal was fired only once
+            if self.run.getpath(request.url) == "/numbers":
+                # Received bytes are not the complete response. The exact amount depends
+                # on the buffer size, which can vary, so we only check that the amount
+                # of received bytes is strictly less than the full response.
+                numbers = [str(x).encode("utf8") for x in range(2**18)]
+                self.assertTrue(len(joined_data) < len(b"".join(numbers)))
diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py
new file mode 100644
index 000000000..fad6643ad
--- /dev/null
+++ b/tests/test_engine_stop_download_headers.py
@@ -0,0 +1,56 @@
+from testfixtures import LogCapture
+from twisted.internet import defer
+
+from scrapy.exceptions import StopDownload
+
+from tests.test_engine import (
+    AttrsItemsSpider,
+    DataClassItemsSpider,
+    DictItemsSpider,
+    TestSpider,
+    CrawlerRun,
+    EngineTest,
+)
+
+
+class HeadersReceivedCrawlerRun(CrawlerRun):
+    def headers_received(self, headers, body_length, request, spider):
+        super().headers_received(headers, body_length, request, spider)
+        raise StopDownload(fail=False)
+
+
+class HeadersReceivedEngineTest(EngineTest):
+    @defer.inlineCallbacks
+    def test_crawler(self):
+        for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
+            if spider is None:
+                continue
+            self.run = HeadersReceivedCrawlerRun(spider)
+            with LogCapture() as log:
+                yield self.run.run()
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> from"
+                                   " signal handler HeadersReceivedCrawlerRun.headers_received"))
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   f"Download stopped for <GET http://localhost:{self.run.portno}/> from signal"
+                                   " handler HeadersReceivedCrawlerRun.headers_received"))
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> from"
+                                   " signal handler HeadersReceivedCrawlerRun.headers_received"))
+            self._assert_visited_urls()
+            self._assert_downloaded_responses(count=6)
+            self._assert_signals_caught()
+            self._assert_bytes_received()
+            self._assert_headers_received()
+
+    def _assert_bytes_received(self):
+        self.assertEqual(0, len(self.run.bytes))
+
+    def _assert_visited_urls(self):
+        must_be_visited = ["/", "/redirect", "/redirected"]
+        urls_visited = {rp[0].url for rp in self.run.respplug}
+        urls_expected = {self.run.geturl(p) for p in must_be_visited}
+        assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"

From 6e5ea7924c7e3f5dd958d13db73e04c6348c99ba Mon Sep 17 00:00:00 2001
From: Dmitriy Pomazunovskiy <dmitriy.pom0@gmail.com>
Date: Fri, 12 Mar 2021 11:08:41 +0600
Subject: [PATCH 312/716] Log skipped urls by length to INFO, add skipped stats

---
 scrapy/spidermiddlewares/urllength.py    | 17 ++++++----
 tests/test_spidermiddleware_urllength.py | 43 +++++++++++++++++++-----
 2 files changed, 45 insertions(+), 15 deletions(-)

diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index 5be1f80cb..ee3cb9fd6 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -14,22 +14,27 @@ logger = logging.getLogger(__name__)
 
 class UrlLengthMiddleware:
 
-    def __init__(self, maxlength):
+    def __init__(self, maxlength, stats):
         self.maxlength = maxlength
+        self.stats = stats
 
     @classmethod
-    def from_settings(cls, settings):
+    def from_crawler(cls, crawler):
+        settings = crawler.settings
         maxlength = settings.getint('URLLENGTH_LIMIT')
         if not maxlength:
             raise NotConfigured
-        return cls(maxlength)
+        return cls(maxlength, crawler.stats)
 
     def process_spider_output(self, response, result, spider):
         def _filter(request):
             if isinstance(request, Request) and len(request.url) > self.maxlength:
-                logger.debug("Ignoring link (url length > %(maxlength)d): %(url)s ",
-                             {'maxlength': self.maxlength, 'url': request.url},
-                             extra={'spider': spider})
+                logger.info(
+                    "Ignoring link (url length > %(maxlength)d): %(url)s ",
+                    {'maxlength': self.maxlength, 'url': request.url},
+                    extra={'spider': spider}
+                )
+                self.stats.inc_value('urllength/request_ignored_count', spider=spider)
                 return False
             else:
                 return True
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index 5ef2b23fd..33c524627 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -1,20 +1,45 @@
 from unittest import TestCase
 
+from testfixtures import LogCapture
+
 from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
+from scrapy.statscollectors import StatsCollector
+from scrapy.utils.test import get_crawler
 
 
 class TestUrlLengthMiddleware(TestCase):
 
-    def test_process_spider_output(self):
-        res = Response('http://scrapytest.org')
+    def setUp(self):
+        crawler = get_crawler(Spider)
+        self.spider = crawler._create_spider('foo')
 
-        short_url_req = Request('http://scrapytest.org/')
-        long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
-        reqs = [short_url_req, long_url_req]
+        self.stats = StatsCollector(crawler)
+        self.stats.open_spider(self.spider)
 
-        mw = UrlLengthMiddleware(maxlength=25)
-        spider = Spider('foo')
-        out = list(mw.process_spider_output(res, reqs, spider))
-        self.assertEqual(out, [short_url_req])
+        self.maxlength = 25
+        self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats)
+
+        self.response = Response('http://scrapytest.org')
+        self.short_url_req = Request('http://scrapytest.org/')
+        self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
+        self.reqs = [self.short_url_req, self.long_url_req]
+
+    def tearDown(self):
+        self.stats.close_spider(self.spider, '')
+
+    def process_spider_output(self):
+        return list(self.mw.process_spider_output(self.response, self.reqs, self.spider))
+
+    def test_middleware_works(self):
+        self.assertEqual(self.process_spider_output(), [self.short_url_req])
+
+    def test_logging(self):
+        with LogCapture() as log:
+            self.process_spider_output()
+
+        ric = self.stats.get_value('urllength/request_ignored_count', spider=self.spider)
+        self.assertEqual(ric, 1)
+
+        self.assertIn(f'Ignoring link (url length > {self.maxlength})', str(log))

From d4b2b612551918647148013893da4cfa83fa2e7a Mon Sep 17 00:00:00 2001
From: Dmitriy Pomazunovskiy <dmitriy.pom0@gmail.com>
Date: Fri, 12 Mar 2021 16:59:37 +0600
Subject: [PATCH 313/716] Use from_settings for backward compatibility

---
 scrapy/spidermiddlewares/urllength.py    | 10 ++++------
 tests/test_spidermiddleware_urllength.py | 10 ++--------
 2 files changed, 6 insertions(+), 14 deletions(-)

diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index ee3cb9fd6..450d4ff40 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -14,17 +14,15 @@ logger = logging.getLogger(__name__)
 
 class UrlLengthMiddleware:
 
-    def __init__(self, maxlength, stats):
+    def __init__(self, maxlength):
         self.maxlength = maxlength
-        self.stats = stats
 
     @classmethod
-    def from_crawler(cls, crawler):
-        settings = crawler.settings
+    def from_settings(cls, settings):
         maxlength = settings.getint('URLLENGTH_LIMIT')
         if not maxlength:
             raise NotConfigured
-        return cls(maxlength, crawler.stats)
+        return cls(maxlength)
 
     def process_spider_output(self, response, result, spider):
         def _filter(request):
@@ -34,7 +32,7 @@ class UrlLengthMiddleware:
                     {'maxlength': self.maxlength, 'url': request.url},
                     extra={'spider': spider}
                 )
-                self.stats.inc_value('urllength/request_ignored_count', spider=spider)
+                spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
                 return False
             else:
                 return True
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index 33c524627..6a72d2a8d 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -5,7 +5,6 @@ from testfixtures import LogCapture
 from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
-from scrapy.statscollectors import StatsCollector
 from scrapy.utils.test import get_crawler
 
 
@@ -14,21 +13,16 @@ class TestUrlLengthMiddleware(TestCase):
     def setUp(self):
         crawler = get_crawler(Spider)
         self.spider = crawler._create_spider('foo')
-
-        self.stats = StatsCollector(crawler)
-        self.stats.open_spider(self.spider)
+        self.stats = self.spider.crawler.stats
 
         self.maxlength = 25
-        self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats)
+        self.mw = UrlLengthMiddleware(maxlength=self.maxlength)
 
         self.response = Response('http://scrapytest.org')
         self.short_url_req = Request('http://scrapytest.org/')
         self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
         self.reqs = [self.short_url_req, self.long_url_req]
 
-    def tearDown(self):
-        self.stats.close_spider(self.spider, '')
-
     def process_spider_output(self):
         return list(self.mw.process_spider_output(self.response, self.reqs, self.spider))
 

From 0f254a6afbc3ad4a42048ea67acafdd035ba690a Mon Sep 17 00:00:00 2001
From: Dmitriy Pomazunovskiy <dmitriy.pom0@gmail.com>
Date: Fri, 12 Mar 2021 17:11:50 +0600
Subject: [PATCH 314/716] Test from_settings

---
 tests/test_spidermiddleware_urllength.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index 6a72d2a8d..ee79c109f 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -6,17 +6,19 @@ from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
+from scrapy.settings import Settings
 
 
 class TestUrlLengthMiddleware(TestCase):
 
     def setUp(self):
+        self.maxlength = 25
+        settings = Settings({'URLLENGTH_LIMIT': self.maxlength})
+
         crawler = get_crawler(Spider)
         self.spider = crawler._create_spider('foo')
         self.stats = self.spider.crawler.stats
-
-        self.maxlength = 25
-        self.mw = UrlLengthMiddleware(maxlength=self.maxlength)
+        self.mw = UrlLengthMiddleware.from_settings(settings)
 
         self.response = Response('http://scrapytest.org')
         self.short_url_req = Request('http://scrapytest.org/')

From c0f3ca193873cd4dbf4de730dcceb38967efcc16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 12 Mar 2021 14:02:48 +0100
Subject: [PATCH 315/716] get_retry_request: add typing information

---
 scrapy/downloadermiddlewares/retry.py | 28 ++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 8955c7e4f..5e49a284a 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -9,8 +9,8 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
 Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
-import logging
-from inspect import isclass
+from logging import getLogger, Logger
+from typing import Optional, Union
 
 from twisted.internet import defer
 from twisted.internet.error import (
@@ -24,24 +24,26 @@ from twisted.internet.error import (
 )
 from twisted.web.client import ResponseFailed
 
-from scrapy.exceptions import NotConfigured
-from scrapy.utils.response import response_status_message
 from scrapy.core.downloader.handlers.http11 import TunnelError
+from scrapy.exceptions import NotConfigured
+from scrapy.http.request import Request
+from scrapy.spiders import Spider
 from scrapy.utils.python import global_object_name
+from scrapy.utils.response import response_status_message
 
 
-retry_logger = logging.getLogger(__name__)
+retry_logger = getLogger(__name__)
 
 
 def get_retry_request(
-    request,
+    request: Request,
     *,
-    spider,
-    reason='unspecified',
-    max_retry_times=None,
-    priority_adjust=None,
-    logger=retry_logger,
-    stats_base_key='retry',
+    spider: Spider,
+    reason: Union[str, Exception] = 'unspecified',
+    max_retry_times: Optional[int] = None,
+    priority_adjust: Union[int, float, None] = None,
+    logger: Logger = retry_logger,
+    stats_base_key: str = 'retry',
 ):
     """
     Returns a new :class:`~scrapy.Request` object to retry the specified
@@ -105,7 +107,7 @@ def get_retry_request(
             priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
         new_request.priority = request.priority + priority_adjust
 
-        if isclass(reason):
+        if callable(reason):
             reason = reason()
         if isinstance(reason, Exception):
             reason = global_object_name(reason.__class__)

From 9cc4513bd60dcebcbfc53035482eff82d2b7acc0 Mon Sep 17 00:00:00 2001
From: Dmitriy Pomazunovskiy <dmitriy.pom0@gmail.com>
Date: Mon, 15 Mar 2021 21:38:03 +0600
Subject: [PATCH 316/716] simpler stats access

---
 tests/test_spidermiddleware_urllength.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index ee79c109f..171f4ddfd 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -17,7 +17,7 @@ class TestUrlLengthMiddleware(TestCase):
 
         crawler = get_crawler(Spider)
         self.spider = crawler._create_spider('foo')
-        self.stats = self.spider.crawler.stats
+        self.stats = crawler.stats
         self.mw = UrlLengthMiddleware.from_settings(settings)
 
         self.response = Response('http://scrapytest.org')

From 2f61d7cc034e6ba6ba6ae9ccfc3cf2f1021b857e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 15 Mar 2021 14:25:46 -0300
Subject: [PATCH 317/716] Remove unnecesary del statement

---
 scrapy/core/http2/agent.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index a142fa210..f7b0c3f99 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -70,8 +70,6 @@ class H2ConnectionPool:
             d = pending_requests.popleft()
             d.callback(conn)
 
-        del pending_requests
-
         return conn
 
     def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None:

From 42e4dbb23de238e1252eb50b059666494418588f Mon Sep 17 00:00:00 2001
From: vinayak <itssvinayak@gmail.com>
Date: Thu, 18 Mar 2021 18:10:03 +0530
Subject: [PATCH 318/716] Support Python 3.9  (#4759)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Update .travis.yml

* Update .travis.yml

* updage travis.yml

* Make 3.9 support official

* Upgrade mitmproxy for Python 3.9

* Restore the Pylint job

* Undo unintended change to mitmproxy requirement

* Enable Python 3.9 in GitHub Actions

* Work around reppy’s Python version limitation

* Disable tests in Windows / Python 3.9 due to a Twisted bug

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 .github/workflows/checks.yml        | 10 ++++++----
 .github/workflows/publish.yml       |  4 ++--
 .github/workflows/tests-macos.yml   |  2 +-
 .github/workflows/tests-ubuntu.yml  |  7 ++++++-
 .github/workflows/tests-windows.yml |  4 ++++
 .readthedocs.yml                    |  6 +++++-
 setup.py                            |  1 +
 tox.ini                             |  4 +++-
 8 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 2748bf5fe..02c647da9 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -7,19 +7,21 @@ jobs:
     strategy:
       matrix:
         include:
-        - python-version: 3.8
+        - python-version: 3.9
           env:
             TOXENV: security
-        - python-version: 3.8
+        - python-version: 3.9
           env:
             TOXENV: flake8
+        # Pylint requires installing reppy, which does not support Python 3.9
+        # https://github.com/seomoz/reppy/issues/122
         - python-version: 3.8
           env:
             TOXENV: pylint
-        - python-version: 3.8
+        - python-version: 3.9
           env:
             TOXENV: typing
-        - python-version: 3.7  # Keep in sync with .readthedocs.yml
+        - python-version: 3.8  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs
 
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index aec6b8696..b48066ea4 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -9,10 +9,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python 3.8
+    - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.9
 
     - name: Check Tag
       id: check-release-tag
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 51d27c405..4f8f7a19d 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -6,7 +6,7 @@ jobs:
     runs-on: macos-10.15
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 89c0334e2..df5ee9d69 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -13,6 +13,9 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: py
+        - python-version: 3.9
+          env:
+            TOXENV: py
         - python-version: pypy3
           env:
             TOXENV: pypy3
@@ -31,10 +34,12 @@ jobs:
             PYPY_VERSION: 3.6-v7.2.0
 
         # extras
+        # extra-deps includes reppy, which does not support Python 3.9
+        # https://github.com/seomoz/reppy/issues/122
         - python-version: 3.8
           env:
             TOXENV: extra-deps
-        - python-version: 3.8
+        - python-version: 3.9
           env:
             TOXENV: asyncio
 
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index ed2e4075d..5459a845b 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -16,6 +16,10 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: py
+        # https://twistedmatrix.com/trac/ticket/9990
+        #- python-version: 3.9
+          #env:
+            #TOXENV: py
 
     steps:
     - uses: actions/checkout@v2
diff --git a/.readthedocs.yml b/.readthedocs.yml
index e4d3f02cc..80a1cd036 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -3,10 +3,14 @@ formats: all
 sphinx:
   configuration: docs/conf.py
   fail_on_warning: true
+
+build:
+  image: latest
+
 python:
   # For available versions, see:
   # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
-  version: 3.7  # Keep in sync with .travis.yml
+  version: 3.8  # Keep in sync with .github/workflows/checks.yml
   install:
     - requirements: docs/requirements.txt
     - path: .
diff --git a/setup.py b/setup.py
index b5c42a3c2..cf9261271 100644
--- a/setup.py
+++ b/setup.py
@@ -85,6 +85,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Internet :: WWW/HTTP',
diff --git a/tox.ini b/tox.ini
index 69f52bd9f..9815f80f7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,7 +13,9 @@ deps =
     -rtests/requirements-py3.txt
     # mitmproxy does not support PyPy
     # mitmproxy does not support Windows when running Python < 3.7
-    mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy'
+    # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe
+    mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
+    mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
     mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # Extras
     botocore>=1.4.87

From 94201612bcad7b74c60a2a7ab70f40ba87714ca8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 18 Mar 2021 23:35:47 +0100
Subject: [PATCH 319/716] Simplify the get_retry_request code example

---
 scrapy/downloadermiddlewares/retry.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 5e49a284a..2721db7cf 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -55,14 +55,12 @@ def get_retry_request(
 
         def parse(self, response):
             if not response.text:
-                new_request = get_retry_request(
+                new_request_or_none = get_retry_request(
                     response.request,
                     spider=self,
                     reason='empty',
                 )
-                if new_request:
-                    yield new_request
-                return
+                return new_request_or_none
 
     *spider* is the :class:`~scrapy.Spider` instance which is asking for the
     retry request. It is used to access the :ref:`settings <topics-settings>`

From 8e73e1dfb51ad6a25ba4583f000d50a12718fb88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 18 Mar 2021 23:42:29 +0100
Subject: [PATCH 320/716] upper-constraints.txt: restrict botocore further

---
 tests/upper-constraints.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/upper-constraints.txt b/tests/upper-constraints.txt
index 75f337856..2a335e533 100644
--- a/tests/upper-constraints.txt
+++ b/tests/upper-constraints.txt
@@ -2,7 +2,7 @@
 # pip dependency resolver from spending too much time backtracking.
 attrs>=20.2.0
 Automat>=0.8.0
-botocore>=1.20.3
+botocore>=1.20.30
 itemadapter>=0.1.1
 itemloaders>=1.0.3
 lxml>=4.6.1

From a390b934de4b8190499c15da115709aa6424d8ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 18 Mar 2021 23:53:58 +0100
Subject: [PATCH 321/716] Do not install mitmproxy in Python 3.9

---
 tox.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 86ae951b5..e0c69350e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,7 +13,8 @@ deps =
     # mitmproxy does not support PyPy
     # mitmproxy does not support Windows when running Python < 3.7
     # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe
-    mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
+    # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
+    #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
     mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
     mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # Extras

From 0dad0fce72266aa7b38b536f87bab26e7f233c74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 19 Mar 2021 11:13:05 +0100
Subject: [PATCH 322/716] Use pip<20.3 to fix ReadTheDocs builds (#5052)

---
 .readthedocs.yml | 1 +
 docs/pip.txt     | 3 +++
 2 files changed, 4 insertions(+)
 create mode 100644 docs/pip.txt

diff --git a/.readthedocs.yml b/.readthedocs.yml
index 80a1cd036..2d781ae81 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -12,5 +12,6 @@ python:
   # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
   version: 3.8  # Keep in sync with .github/workflows/checks.yml
   install:
+    - requirements: docs/pip.txt
     - requirements: docs/requirements.txt
     - path: .
diff --git a/docs/pip.txt b/docs/pip.txt
new file mode 100644
index 000000000..095e53a0d
--- /dev/null
+++ b/docs/pip.txt
@@ -0,0 +1,3 @@
+# In pip 20.3-21.0, the default dependency resolver causes the build in
+# ReadTheDocs to fail due to memory exhaustion or timeout.
+pip<20.3

From 308a58aa275aa514397351caa263e08de2f89adc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 19 Mar 2021 18:39:44 +0100
Subject: [PATCH 323/716] Update CI to support Twisted 21.2.0 (#5027)

---
 scrapy/pipelines/images.py    | 19 +++++++++++++------
 tests/test_commands.py        |  3 +++
 tests/test_crawler.py         |  4 ++++
 tests/test_pipeline_crawl.py  | 11 +++++++++++
 tests/test_pipeline_images.py | 14 ++++++++++----
 tests/test_pipeline_media.py  | 11 +++++++++++
 tox.ini                       |  4 +---
 7 files changed, 53 insertions(+), 13 deletions(-)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index aafd1d8b2..e3ab23ea5 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -9,9 +9,8 @@ from contextlib import suppress
 from io import BytesIO
 
 from itemadapter import ItemAdapter
-from PIL import Image
 
-from scrapy.exceptions import DropItem
+from scrapy.exceptions import DropItem, NotConfigured
 from scrapy.http import Request
 from scrapy.pipelines.files import FileException, FilesPipeline
 # TODO: from scrapy.pipelines.media import MediaPipeline
@@ -45,6 +44,14 @@ class ImagesPipeline(FilesPipeline):
     DEFAULT_IMAGES_RESULT_FIELD = 'images'
 
     def __init__(self, store_uri, download_func=None, settings=None):
+        try:
+            from PIL import Image
+            self._Image = Image
+        except ImportError:
+            raise NotConfigured(
+                'ImagesPipeline requires installing Pillow 4.0.0 or later'
+            )
+
         super().__init__(store_uri, settings=settings, download_func=download_func)
 
         if isinstance(settings, dict) or settings is None:
@@ -121,7 +128,7 @@ class ImagesPipeline(FilesPipeline):
 
     def get_images(self, response, request, info, *, item=None):
         path = self.file_path(request, response=response, info=info, item=item)
-        orig_image = Image.open(BytesIO(response.body))
+        orig_image = self._Image.open(BytesIO(response.body))
 
         width, height = orig_image.size
         if width < self.min_width or height < self.min_height:
@@ -139,12 +146,12 @@ class ImagesPipeline(FilesPipeline):
 
     def convert_image(self, image, size=None):
         if image.format == 'PNG' and image.mode == 'RGBA':
-            background = Image.new('RGBA', image.size, (255, 255, 255))
+            background = self._Image.new('RGBA', image.size, (255, 255, 255))
             background.paste(image, image)
             image = background.convert('RGB')
         elif image.mode == 'P':
             image = image.convert("RGBA")
-            background = Image.new('RGBA', image.size, (255, 255, 255))
+            background = self._Image.new('RGBA', image.size, (255, 255, 255))
             background.paste(image, image)
             image = background.convert('RGB')
         elif image.mode != 'RGB':
@@ -152,7 +159,7 @@ class ImagesPipeline(FilesPipeline):
 
         if size:
             image = image.copy()
-            image.thumbnail(size, Image.ANTIALIAS)
+            image.thumbnail(size, self._Image.ANTIALIAS)
 
         buf = BytesIO()
         image.save(buf, 'JPEG')
diff --git a/tests/test_commands.py b/tests/test_commands.py
index d3ac05eac..eec1f02ee 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -17,6 +17,8 @@ from threading import Timer
 from unittest import skipIf
 
 from pytest import mark
+from twisted import version as twisted_version
+from twisted.python.versions import Version
 from twisted.trial import unittest
 
 import scrapy
@@ -630,6 +632,7 @@ class MySpider(scrapy.Spider):
 
     @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
     @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
     def test_custom_asyncio_loop_enabled_true(self):
         log = self.get_log(self.debug_log_spider, args=[
             '-s',
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index ab113710d..dec517bb6 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -8,7 +8,9 @@ from unittest import skipIf
 
 from pytest import raises, mark
 from testfixtures import LogCapture
+from twisted import version as twisted_version
 from twisted.internet import defer
+from twisted.python.versions import Version
 from twisted.trial import unittest
 
 import scrapy
@@ -358,6 +360,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
 
     @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
     @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
     def test_custom_loop_asyncio(self):
         log = self.run_script("asyncio_custom_loop.py")
         self.assertIn("Spider closed (finished)", log)
@@ -366,6 +369,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
 
     @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly")
     @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows")
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
     def test_custom_loop_asyncio_deferred_signal(self):
         log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
         self.assertIn("Spider closed (finished)", log)
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 55fcfa7ba..f49fda701 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -180,7 +180,18 @@ class FileDownloadCrawlTestCase(TestCase):
         self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)
 
 
+try:
+    from PIL import Image  # noqa: imported just to check for the import error
+except ImportError:
+    skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+else:
+    skip_pillow = None
+
+
 class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase):
+
+    skip = skip_pillow
+
     pipeline_class = 'scrapy.pipelines.images.ImagesPipeline'
     store_setting_key = 'IMAGES_STORE'
     media_key = 'images'
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index ad138a2dc..c69cd0e4a 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -23,15 +23,16 @@ except ImportError:
     dataclass_field = None
 
 
-skip = False
 try:
     from PIL import Image
 except ImportError:
-    skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+    skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
 else:
     encoders = {'jpeg_encoder', 'jpeg_decoder'}
     if not encoders.issubset(set(Image.core.__dict__)):
-        skip = 'Missing JPEG encoders'
+        skip_pillow = 'Missing JPEG encoders'
+    else:
+        skip_pillow = None
 
 
 def _mocked_download_func(request, info):
@@ -41,7 +42,7 @@ def _mocked_download_func(request, info):
 
 class ImagesPipelineTestCase(unittest.TestCase):
 
-    skip = skip
+    skip = skip_pillow
 
     def setUp(self):
         self.tempdir = mkdtemp()
@@ -137,6 +138,8 @@ class DeprecatedImagesPipeline(ImagesPipeline):
 
 class ImagesPipelineTestCaseFieldsMixin:
 
+    skip = skip_pillow
+
     def test_item_fields_default(self):
         url = 'http://www.example.com/images/1.jpg'
         item = self.item_class(name='item1', image_urls=[url])
@@ -221,6 +224,9 @@ class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, u
 
 
 class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
+
+    skip = skip_pillow
+
     img_cls_attribute_names = [
         # Pipeline attribute names with corresponding setting names.
         ("EXPIRES", "IMAGES_EXPIRES"),
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 6afd47497..893d43052 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,3 +1,5 @@
+from typing import Optional
+
 from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
@@ -17,6 +19,14 @@ from scrapy.utils.signal import disconnect_all
 from scrapy import signals
 
 
+try:
+    from PIL import Image  # noqa: imported just to check for the import error
+except ImportError:
+    skip_pillow: Optional[str] = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+else:
+    skip_pillow = None
+
+
 def _mocked_download_func(request, info):
     response = request.meta.get('response')
     return response() if callable(response) else response
@@ -379,6 +389,7 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
 
 
 class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
+    skip = skip_pillow
 
     def setUp(self):
         self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func)
diff --git a/tox.ini b/tox.ini
index e0c69350e..6907c8906 100644
--- a/tox.ini
+++ b/tox.ini
@@ -19,9 +19,6 @@ deps =
     mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # Extras
     botocore>=1.4.87
-    Pillow>=4.0.0
-    # Twisted 21+ causes issues in tests that use skipIf
-    Twisted[http2]>=17.9.0,<21
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -124,6 +121,7 @@ deps =
     {[testenv]deps}
     reppy
     robotexclusionrulesparser
+    Pillow>=4.0.0
 
 [testenv:asyncio]
 commands =

From 2973d8d51abbda4839981c192a366e907e2ad39e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 22 Mar 2021 11:24:10 +0100
Subject: [PATCH 324/716] Remove unnecessary reference to private
 parsel.Selector._default_type (#5006)

---
 scrapy/selector/unified.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index a25871433..08f08e8d7 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -69,7 +69,7 @@ class Selector(_ParselSelector, object_ref):
             raise ValueError(f'{self.__class__.__name__}.__init__() received '
                              'both response and text')
 
-        st = _st(response, type or self._default_type)
+        st = _st(response, type)
 
         if text is not None:
             response = _response_from_text(text, st)

From ec5a7918ec2d8888ba356f9e3295dcd1ee935884 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 22 Mar 2021 11:25:40 +0100
Subject: [PATCH 325/716] Include Content-Length in HTTP/1.1 responses (#5057)

---
 scrapy/core/downloader/handlers/http11.py | 13 +++++++++++--
 tests/test_downloader_handlers.py         |  7 +++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 9cdadb27f..1f82751fd 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -358,10 +358,19 @@ class ScrapyAgent:
         request.meta['download_latency'] = time() - start_time
         return result
 
+    @staticmethod
+    def _headers_from_twisted_response(response):
+        headers = Headers()
+        if response.length is not None:
+            headers[b'Content-Length'] = str(response.length).encode()
+        for key, value in response.headers.getAllRawHeaders():
+            headers[key] = value
+        return headers
+
     def _cb_bodyready(self, txresponse, request):
         headers_received_result = self._crawler.signals.send_catch_log(
             signal=signals.headers_received,
-            headers=Headers(txresponse.headers.getAllRawHeaders()),
+            headers=self._headers_from_twisted_response(txresponse),
             body_length=txresponse.length,
             request=request,
             spider=self._crawler.spider,
@@ -435,7 +444,7 @@ class ScrapyAgent:
         return d
 
     def _cb_bodydone(self, result, request, url):
-        headers = Headers(result["txresponse"].headers.getAllRawHeaders())
+        headers = self._headers_from_twisted_response(result["txresponse"])
         respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
         try:
             version = result["txresponse"].version
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 86d72772c..fa7d5c8a6 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -364,6 +364,13 @@ class HttpTestCase(unittest.TestCase):
         d.addCallback(self.assertEqual, body)
         return d
 
+    def test_response_header_content_length(self):
+        request = Request(self.getURL("file"), method=b"GET")
+        d = self.download_request(request, Spider("foo"))
+        d.addCallback(lambda r: r.headers[b'content-length'])
+        d.addCallback(self.assertEqual, b'159')
+        return d
+
 
 class Http10TestCase(HttpTestCase):
     """HTTP 1.0 test case"""

From 72e8cea8afb85f6704fcf6f5648b0a01f1abaab3 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Mon, 22 Mar 2021 11:51:11 -0300
Subject: [PATCH 326/716] Avoid exceptions in is_generator_with_return_value
 (#4935)

---
 scrapy/utils/misc.py                          |  27 ++-
 ...t_return_with_argument_inside_generator.py | 167 +++++++++++++++---
 2 files changed, 161 insertions(+), 33 deletions(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 081cd33f1..5c986eedc 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -9,7 +9,6 @@ from collections import deque
 from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
-from textwrap import dedent
 
 from w3lib.html import replace_entities
 
@@ -227,7 +226,8 @@ def is_generator_with_return_value(callable):
         return value is None or isinstance(value, ast.NameConstant) and value.value is None
 
     if inspect.isgeneratorfunction(callable):
-        tree = ast.parse(dedent(inspect.getsource(callable)))
+        code = re.sub(r"^[\t ]+", "", inspect.getsource(callable))
+        tree = ast.parse(code)
         for node in walk_callable(tree):
             if isinstance(node, ast.Return) and not returns_none(node):
                 _generator_callbacks_cache[callable] = True
@@ -242,12 +242,23 @@ def warn_on_generator_with_return_value(spider, callable):
     Logs a warning if a callable is a generator function and includes
     a 'return' statement with a value different than None
     """
-    if is_generator_with_return_value(callable):
+    try:
+        if is_generator_with_return_value(callable):
+            warnings.warn(
+                f'The "{spider.__class__.__name__}.{callable.__name__}" method is '
+                'a generator and includes a "return" statement with a value '
+                'different than None. This could lead to unexpected behaviour. Please see '
+                'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '
+                'for details about the semantics of the "return" statement within generators',
+                stacklevel=2,
+            )
+    except IndentationError:
+        callable_name = spider.__class__.__name__ + "." + callable.__name__
         warnings.warn(
-            f'The "{spider.__class__.__name__}.{callable.__name__}" method is '
-            'a generator and includes a "return" statement with a value '
-            'different than None. This could lead to unexpected behaviour. Please see '
-            'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '
-            'for details about the semantics of the "return" statement within generators',
+            f'Unable to determine whether or not "{callable_name}" is a generator with a return value. '
+            'This will not prevent your code from working, but it prevents Scrapy from detecting '
+            f'potential issues in your implementation of "{callable_name}". Please, report this in the '
+            'Scrapy issue tracker (https://github.com/scrapy/scrapy/issues), '
+            f'including the code of "{callable_name}"',
             stacklevel=2,
         )
diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
index 2be38620c..1c85ca353 100644
--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py
+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
@@ -1,35 +1,116 @@
 import unittest
+import warnings
+from unittest import mock
 
-from scrapy.utils.misc import is_generator_with_return_value
+from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value
+
+
+def _indentation_error(*args, **kwargs):
+    raise IndentationError()
+
+
+def top_level_return_something():
+    """
+docstring
+    """
+    url = """
+https://example.org
+"""
+    yield url
+    return 1
+
+
+def top_level_return_none():
+    """
+docstring
+    """
+    url = """
+https://example.org
+"""
+    yield url
+    return
+
+
+def generator_that_returns_stuff():
+    yield 1
+    yield 2
+    return 3
 
 
 class UtilsMiscPy3TestCase(unittest.TestCase):
 
-    def test_generators_with_return_statements(self):
-        def f():
+    def test_generators_return_something(self):
+        def f1():
             yield 1
             return 2
 
-        def g():
+        def g1():
             yield 1
-            return 'asdf'
+            return "asdf"
 
-        def h():
+        def h1():
+            yield 1
+
+            def helper():
+                return 0
+
+            yield helper()
+            return 2
+
+        def i1():
+            """
+docstring
+            """
+            url = """
+https://example.org
+        """
+            yield url
+            return 1
+
+        assert is_generator_with_return_value(top_level_return_something)
+        assert is_generator_with_return_value(f1)
+        assert is_generator_with_return_value(g1)
+        assert is_generator_with_return_value(h1)
+        assert is_generator_with_return_value(i1)
+
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, top_level_return_something)
+            self.assertEqual(len(w), 1)
+            self.assertIn('The "NoneType.top_level_return_something" method is a generator', str(w[0].message))
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, f1)
+            self.assertEqual(len(w), 1)
+            self.assertIn('The "NoneType.f1" method is a generator', str(w[0].message))
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, g1)
+            self.assertEqual(len(w), 1)
+            self.assertIn('The "NoneType.g1" method is a generator', str(w[0].message))
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, h1)
+            self.assertEqual(len(w), 1)
+            self.assertIn('The "NoneType.h1" method is a generator', str(w[0].message))
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, i1)
+            self.assertEqual(len(w), 1)
+            self.assertIn('The "NoneType.i1" method is a generator', str(w[0].message))
+
+    def test_generators_return_none(self):
+        def f2():
             yield 1
             return None
 
-        def i():
+        def g2():
             yield 1
             return
 
-        def j():
+        def h2():
             yield 1
 
-        def k():
+        def i2():
             yield 1
-            yield from g()
+            yield from generator_that_returns_stuff()
 
-        def m():
+        def j2():
             yield 1
 
             def helper():
@@ -37,20 +118,56 @@ class UtilsMiscPy3TestCase(unittest.TestCase):
 
             yield helper()
 
-        def n():
-            yield 1
+        def k2():
+            """
+docstring
+            """
+            url = """
+https://example.org
+        """
+            yield url
+            return
 
-            def helper():
-                return 0
+        def l2():
+            return
 
-            yield helper()
-            return 2
+        assert not is_generator_with_return_value(top_level_return_none)
+        assert not is_generator_with_return_value(f2)
+        assert not is_generator_with_return_value(g2)
+        assert not is_generator_with_return_value(h2)
+        assert not is_generator_with_return_value(i2)
+        assert not is_generator_with_return_value(j2)  # not recursive
+        assert not is_generator_with_return_value(k2)  # not recursive
+        assert not is_generator_with_return_value(l2)
 
-        assert is_generator_with_return_value(f)
-        assert is_generator_with_return_value(g)
-        assert not is_generator_with_return_value(h)
-        assert not is_generator_with_return_value(i)
-        assert not is_generator_with_return_value(j)
-        assert not is_generator_with_return_value(k)  # not recursive
-        assert not is_generator_with_return_value(m)
-        assert is_generator_with_return_value(n)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, top_level_return_none)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, f2)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, g2)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, h2)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, i2)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, j2)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, k2)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, l2)
+            self.assertEqual(len(w), 0)
+
+    @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error)
+    def test_indentation_error(self):
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, top_level_return_none)
+            self.assertEqual(len(w), 1)
+            self.assertIn('Unable to determine', str(w[0].message))

From 64d4ae1a19eda9c6bb0b6dc85e5dfff6187d0ca8 Mon Sep 17 00:00:00 2001
From: Marc <noviluni@gmail.com>
Date: Mon, 22 Mar 2021 21:46:05 +0100
Subject: [PATCH 327/716] Update UsageError message

---
 scrapy/commands/crawl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index f205c40b0..0f2a21b85 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand):
         if len(args) < 1:
             raise UsageError()
         elif len(args) > 1:
-            raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
+            raise UsageError("running 'scrapy crawl' with more than one spider is not supported")
         spname = args[0]
 
         crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)

From f0e1a33225dd7c9162e277626df70284f41a427e Mon Sep 17 00:00:00 2001
From: Pratik Mahankal <53421565+pratik1500@users.noreply.github.com>
Date: Tue, 23 Mar 2021 22:46:50 +0530
Subject: [PATCH 328/716] Sort the list of Request.meta alphabetically #5061
 (#5065)

---
 docs/topics/request-response.rst | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index a0448c5ab..c0283df01 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -363,26 +363,26 @@ are some special keys recognized by Scrapy and its built-in extensions.
 
 Those are:
 
-* :reqmeta:`dont_redirect`
-* :reqmeta:`dont_retry`
-* :reqmeta:`handle_httpstatus_list`
-* :reqmeta:`handle_httpstatus_all`
-* :reqmeta:`dont_merge_cookies`
+* :reqmeta:`bindaddress`
 * :reqmeta:`cookiejar`
 * :reqmeta:`dont_cache`
+* :reqmeta:`dont_merge_cookies`
+* :reqmeta:`dont_obey_robotstxt`
+* :reqmeta:`dont_redirect`
+* :reqmeta:`dont_retry`
+* :reqmeta:`download_fail_on_dataloss`
+* :reqmeta:`download_latency`
+* :reqmeta:`download_maxsize`
+* :reqmeta:`download_timeout`
+* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
+* ``ftp_user`` (See :setting:`FTP_USER` for more info)
+* :reqmeta:`handle_httpstatus_all`
+* :reqmeta:`handle_httpstatus_list`
+* :reqmeta:`max_retry_times`
+* :reqmeta:`proxy`
 * :reqmeta:`redirect_reasons`
 * :reqmeta:`redirect_urls`
-* :reqmeta:`bindaddress`
-* :reqmeta:`dont_obey_robotstxt`
-* :reqmeta:`download_timeout`
-* :reqmeta:`download_maxsize`
-* :reqmeta:`download_latency`
-* :reqmeta:`download_fail_on_dataloss`
-* :reqmeta:`proxy`
-* ``ftp_user`` (See :setting:`FTP_USER` for more info)
-* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
 * :reqmeta:`referrer_policy`
-* :reqmeta:`max_retry_times`
 
 .. reqmeta:: bindaddress
 

From c51ec1ae1cef3fd68fc512f0636b75d0f3a2da13 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 26 Feb 2021 19:32:45 +0500
Subject: [PATCH 329/716] Drop process_iterable_helper, add
 _process_iterable_universal.

---
 scrapy/core/spidermw.py               | 28 +++------
 scrapy/spidermiddlewares/depth.py     | 18 ++++--
 scrapy/spidermiddlewares/offsite.py   | 32 +++++-----
 scrapy/spidermiddlewares/referer.py   |  9 ++-
 scrapy/spidermiddlewares/urllength.py |  9 ++-
 scrapy/utils/asyncgen.py              | 45 ++++++++++++++
 scrapy/utils/middlewares.py           | 35 -----------
 tests/test_utils_asyncgen.py          | 22 ++++++-
 tests/test_utils_middlewares.py       | 87 ---------------------------
 9 files changed, 120 insertions(+), 165 deletions(-)
 delete mode 100644 scrapy/utils/middlewares.py
 delete mode 100644 tests/test_utils_middlewares.py

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index d3d7e5f8c..33e215971 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -10,6 +10,7 @@ from twisted.python.failure import Failure
 
 from scrapy.exceptions import _InvalidOutput
 from scrapy.middleware import MiddlewareManager
+from scrapy.utils.asyncgen import _process_iterable_universal
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.python import MutableAsyncChain, MutableChain
@@ -57,31 +58,18 @@ class SpiderMiddlewareManager(MiddlewareManager):
         return scrape_func(response, request, spider)
 
     def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to):
-        def _process_exception(ex):
-            exception_result = self._process_spider_exception(response, spider, Failure(ex),
-                                                              exception_processor_index)
-            if isinstance(exception_result, Failure):
-                raise  # pylint: disable=E0704
-            recover_to.extend(exception_result)
-
-        def _evaluate_normal_iterable(iterable):
-            try:
-                for r in iterable:
-                    yield r
-            except Exception as ex:
-                _process_exception(ex)
-
+        @_process_iterable_universal
         async def _evaluate_async_iterable(iterable):
             try:
                 async for r in iterable:
                     yield r
             except Exception as ex:
-                _process_exception(ex)
-
-        if inspect.isasyncgen(iterable):
-            return _evaluate_async_iterable(iterable)
-        else:
-            return _evaluate_normal_iterable(iterable)
+                exception_result = self._process_spider_exception(response, spider, Failure(ex),
+                                                                  exception_processor_index)
+                if isinstance(exception_result, Failure):
+                    raise
+                recover_to.extend(exception_result)
+        return _evaluate_async_iterable(iterable)
 
     def _process_spider_exception(self, response, spider, _failure, start_index=0):
         exception = _failure.value
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 776a6879a..973404b2b 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -7,6 +7,7 @@ See documentation in docs/topics/spider-middleware.rst
 import logging
 
 from scrapy.http import Request
+from scrapy.utils.asyncgen import _process_iterable_universal
 
 logger = logging.getLogger(__name__)
 
@@ -49,10 +50,15 @@ class DepthMiddleware:
                                          spider=spider)
             return True
 
-        # base case (depth=0)
-        if 'depth' not in response.meta:
-            response.meta['depth'] = 0
-            if self.verbose_stats:
-                self.stats.inc_value('request_depth_count/0', spider=spider)
+        @_process_iterable_universal
+        async def process(result):
+            # base case (depth=0)
+            if 'depth' not in response.meta:
+                response.meta['depth'] = 0
+                if self.verbose_stats:
+                    self.stats.inc_value('request_depth_count/0', spider=spider)
 
-        return (r for r in result or () if _filter(r))
+            async for r in result or ():
+                if _filter(r):
+                    yield r
+        return process(result)
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 6e4efda97..074ec7a4e 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -9,6 +9,7 @@ import warnings
 
 from scrapy import signals
 from scrapy.http import Request
+from scrapy.utils.asyncgen import _process_iterable_universal
 from scrapy.utils.httpobj import urlparse_cached
 
 logger = logging.getLogger(__name__)
@@ -26,21 +27,24 @@ class OffsiteMiddleware:
         return o
 
     def process_spider_output(self, response, result, spider):
-        for x in result:
-            if isinstance(x, Request):
-                if x.dont_filter or self.should_follow(x, spider):
-                    yield x
+        @_process_iterable_universal
+        async def process(result):
+            async for x in result:
+                if isinstance(x, Request):
+                    if x.dont_filter or self.should_follow(x, spider):
+                        yield x
+                    else:
+                        domain = urlparse_cached(x).hostname
+                        if domain and domain not in self.domains_seen:
+                            self.domains_seen.add(domain)
+                            logger.debug(
+                                "Filtered offsite request to %(domain)r: %(request)s",
+                                {'domain': domain, 'request': x}, extra={'spider': spider})
+                            self.stats.inc_value('offsite/domains', spider=spider)
+                        self.stats.inc_value('offsite/filtered', spider=spider)
                 else:
-                    domain = urlparse_cached(x).hostname
-                    if domain and domain not in self.domains_seen:
-                        self.domains_seen.add(domain)
-                        logger.debug(
-                            "Filtered offsite request to %(domain)r: %(request)s",
-                            {'domain': domain, 'request': x}, extra={'spider': spider})
-                        self.stats.inc_value('offsite/domains', spider=spider)
-                    self.stats.inc_value('offsite/filtered', spider=spider)
-            else:
-                yield x
+                    yield x
+        return process(result)
 
     def should_follow(self, request, spider):
         regex = self.host_regex
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index f81041376..8d862d1d0 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -10,6 +10,7 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
+from scrapy.utils.asyncgen import _process_iterable_universal
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
@@ -337,7 +338,13 @@ class RefererMiddleware:
                 if referrer is not None:
                     r.headers.setdefault('Referer', referrer)
             return r
-        return (_set_referer(r) for r in result or ())
+
+        @_process_iterable_universal
+        async def process(result):
+            async for r in result or ():
+                yield _set_referer(r)
+
+        return process(result)
 
     def request_scheduled(self, request, spider):
         # check redirected request to patch "Referer" header if necessary
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index 5be1f80cb..d40d43ff1 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -8,6 +8,7 @@ import logging
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
+from scrapy.utils.asyncgen import _process_iterable_universal
 
 logger = logging.getLogger(__name__)
 
@@ -34,4 +35,10 @@ class UrlLengthMiddleware:
             else:
                 return True
 
-        return (r for r in result or () if _filter(r))
+        @_process_iterable_universal
+        async def process(result):
+            async for r in result or ():
+                if _filter(r):
+                    yield r
+
+        return process(result)
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index db2173f85..39c94ad8a 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,4 +1,6 @@
 import collections
+import functools
+import inspect
 
 
 async def collect_asyncgen(result):
@@ -15,3 +17,46 @@ async def as_async_generator(it):
     else:
         for r in it:
             yield r
+
+
+# https://stackoverflow.com/a/66170760/113586
+def _process_iterable_universal(process_async):
+    """ Takes a function that takes an async iterable, args and kwargs. Returns
+    a function that takes any iterable, args and kwargs.
+
+    Requires that process_async only awaits on the iterable and synchronous functions,
+    so it's better to use this only in the Scrapy code itself.
+    """
+
+    # If this stops working, all internal uses can be just replaced with manually-written
+    # process_sync functions.
+
+    def process_sync(iterable, *args, **kwargs):
+        agen = process_async(as_async_generator(iterable), *args, **kwargs)
+        if not inspect.isasyncgen(agen):
+            raise ValueError(f"process_async returned wrong type {type(agen)}")
+        sent = None
+        while True:
+            try:
+                gen = agen.asend(sent)
+                gen.send(None)
+            except StopIteration as e:
+                sent = yield e.value
+            except StopAsyncIteration:
+                return
+            else:
+                gen.throw(RuntimeError,
+                          f"Synchronously-called function '{process_async.__name__}' has blocked, "
+                          f"you can't use {_process_iterable_universal.__name__} with it.")
+
+    @functools.wraps(process_async)
+    def process(iterable, *args, **kwargs):
+        if inspect.isasyncgen(iterable):
+            # call process_async directly
+            return process_async(iterable, *args, **kwargs)
+        if hasattr(iterable, '__iter__'):
+            # convert process_async to process_sync
+            return process_sync(iterable, *args, **kwargs)
+        raise ValueError(f"Wrong iterable type {type(iterable)}")
+
+    return process
diff --git a/scrapy/utils/middlewares.py b/scrapy/utils/middlewares.py
deleted file mode 100644
index da28e0ddf..000000000
--- a/scrapy/utils/middlewares.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# coding: utf-8
-import inspect
-
-
-def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
-    for o in it:
-        if in_predicate and not in_predicate(o):
-            continue
-        if processor is not None:
-            o = processor(o)
-        if out_predicate and not out_predicate(o):
-            continue
-        yield o
-
-
-async def process_async_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
-    async for o in it:
-        if in_predicate and not in_predicate(o):
-            continue
-        if processor is not None:
-            o = processor(o)
-        if out_predicate and not out_predicate(o):
-            continue
-        yield o
-
-
-def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
-    """
-    For each item in the iterable: skips it if in_predicate is False, applies processor,
-    skips the result if out_predicate is False, else yields it.
-    """
-    if inspect.isasyncgen(it):
-        return process_async_iterable_helper(it, in_predicate, out_predicate, processor)
-    else:
-        return process_normal_iterable_helper(it, in_predicate, out_predicate, processor)
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index 9ae66c57c..2f4181d3d 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -1,6 +1,6 @@
 from twisted.trial import unittest
 
-from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal
 from scrapy.utils.defer import deferred_f_from_coro_f
 
 
@@ -18,3 +18,23 @@ class AsyncgenUtilsTest(unittest.TestCase):
         ag = as_async_generator(range(42))
         results = await collect_asyncgen(ag)
         self.assertEqual(results, list(range(42)))
+
+
+@_process_iterable_universal
+async def process_iterable(iterable):
+    async for i in iterable:
+        yield i * 2
+
+
+class ProcessIterableUniversalTest(unittest.TestCase):
+
+    def test_normal(self):
+        iterable = iter([1, 2, 3])
+        results = list(process_iterable(iterable))
+        self.assertEqual(results, [2, 4, 6])
+
+    @deferred_f_from_coro_f
+    async def test_async(self):
+        iterable = as_async_generator([1, 2, 3])
+        results = await collect_asyncgen(process_iterable(iterable))
+        self.assertEqual(results, [2, 4, 6])
diff --git a/tests/test_utils_middlewares.py b/tests/test_utils_middlewares.py
deleted file mode 100644
index d395ba1a9..000000000
--- a/tests/test_utils_middlewares.py
+++ /dev/null
@@ -1,87 +0,0 @@
-import collections
-
-from twisted.trial import unittest
-
-from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
-from scrapy.utils.defer import deferred_f_from_coro_f
-from scrapy.utils.middlewares import process_iterable_helper
-
-
-def predicate1(o):
-    return bool(o % 2)
-
-
-def predicate2(o):
-    return o < 10
-
-
-def processor(o):
-    return o * 2
-
-
-class ProcessIterableHelperNormalTest(unittest.TestCase):
-
-    def test_normal_in_predicate(self):
-        iterable1 = iter([1, 2, 3])
-        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1)
-        self.assertIsInstance(iterable2, collections.abc.Iterable)
-        list2 = list(iterable2)
-        self.assertEqual(list2, [1, 3])
-
-    def test_normal_out_predicate(self):
-        iterable1 = iter([1, 2, 10, 3, 15])
-        iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2)
-        self.assertIsInstance(iterable2, collections.abc.Iterable)
-        list2 = list(iterable2)
-        self.assertEqual(list2, [1, 2, 3])
-
-    def test_normal_processor(self):
-        iterable1 = iter([1, 2, 3])
-        iterable2 = process_iterable_helper(iterable1, processor=processor)
-        self.assertIsInstance(iterable2, collections.abc.Iterable)
-        list2 = list(iterable2)
-        self.assertEqual(list2, [2, 4, 6])
-
-    def test_normal_combined(self):
-        iterable1 = iter([1, 2, 10, 3, 6, 18, 5, 15])
-        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1,
-                                            out_predicate=predicate2, processor=processor)
-        self.assertIsInstance(iterable2, collections.abc.Iterable)
-        list2 = list(iterable2)
-        self.assertEqual(list2, [2, 6])
-
-
-class ProcessIterableHelperAsyncTest(unittest.TestCase):
-
-    @deferred_f_from_coro_f
-    async def test_async_in_predicate(self):
-        iterable1 = as_async_generator([1, 2, 3])
-        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1)
-        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
-        list2 = await collect_asyncgen(iterable2)
-        self.assertEqual(list2, [1, 3])
-
-    @deferred_f_from_coro_f
-    async def test_async_out_predicate(self):
-        iterable1 = as_async_generator([1, 2, 10, 3, 15])
-        iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2)
-        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
-        list2 = await collect_asyncgen(iterable2)
-        self.assertEqual(list2, [1, 2, 3])
-
-    @deferred_f_from_coro_f
-    async def test_async_processor(self):
-        iterable1 = as_async_generator([1, 2, 3])
-        iterable2 = process_iterable_helper(iterable1, processor=processor)
-        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
-        list2 = await collect_asyncgen(iterable2)
-        self.assertEqual(list2, [2, 4, 6])
-
-    @deferred_f_from_coro_f
-    async def test_async_combined(self):
-        iterable1 = as_async_generator([1, 2, 10, 3, 6, 18, 5, 15])
-        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1,
-                                            out_predicate=predicate2, processor=processor)
-        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
-        list2 = await collect_asyncgen(iterable2)
-        self.assertEqual(list2, [2, 6])

From a6034f942b034946538855cc53644b5a89ba081a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 22 Mar 2021 22:53:52 +0500
Subject: [PATCH 330/716] Add tests for _AsyncCooperatorAdapter.

---
 tests/test_utils_defer.py | 68 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 67 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 62f6ff194..543bbee09 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -1,14 +1,18 @@
+import random
+
 from pytest import mark
 from twisted.trial import unittest
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
-from scrapy.utils.asyncgen import collect_asyncgen
+from scrapy.utils.asyncgen import collect_asyncgen, as_async_generator
 from scrapy.utils.defer import (
     aiter_errback,
     deferred_f_from_coro_f,
     iter_errback,
+    maybe_deferred_to_future,
     mustbe_deferred,
+    parallel_async,
     process_chain,
     process_chain_both,
     process_parallel,
@@ -164,3 +168,65 @@ class AsyncDefTestsuiteTest(unittest.TestCase):
     @deferred_f_from_coro_f
     async def test_deferred_f_from_coro_f_xfail(self):
         raise Exception("This is expected to be raised")
+
+
+class AsyncCooperatorTest(unittest.TestCase):
+    """ This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.
+
+    parallel_async is called with the results of a callback (so an iterable of items, requests and None,
+    with arbitrary delays between values), and it uses Scraper._process_spidermw_output as the callable
+    (so a callable that returns a Deferred for an item, which will fire after pipelines process it, and
+    None for everything else). The concurrent task count is the CONCURRENT_ITEMS setting.
+
+    We want to test different concurrency values compared to the iterable length.
+    We also want to simulate the real usage, with arbitrary delays between getting the values
+    from the iterable. We also want to simulate sync and async results from the callable.
+    """
+    CONCURRENT_ITEMS = 50
+
+    @staticmethod
+    def callable(o, results):
+        if random.random() < 0.4:
+            # simulate async processing
+            dfd = defer.Deferred()
+            dfd.addCallback(lambda _: results.append(o))
+            delay = random.random() / 8
+            reactor.callLater(delay, dfd.callback, None)
+            return dfd
+        else:
+            # simulate trivial sync processing
+            results.append(o)
+
+    @staticmethod
+    def get_async_iterable(length):
+        # simulate a simple callback without delays between results
+        return as_async_generator(range(length))
+
+    @staticmethod
+    async def get_async_iterable_with_delays(length):
+        # simulate a callback with delays between some of the results
+        for i in range(length):
+            if random.random() < 0.1:
+                dfd = defer.Deferred()
+                delay = random.random() / 20
+                reactor.callLater(delay, dfd.callback, None)
+                await maybe_deferred_to_future(dfd)
+            yield i
+
+    @defer.inlineCallbacks
+    def test_simple(self):
+        for length in [20, 50, 100]:
+            results = []
+            ait = self.get_async_iterable(length)
+            dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results)
+            yield dl
+            self.assertEqual(list(range(length)), sorted(results))
+
+    @defer.inlineCallbacks
+    def test_delays(self):
+        for length in [20, 50, 100]:
+            results = []
+            ait = self.get_async_iterable_with_delays(length)
+            dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results)
+            yield dl
+            self.assertEqual(list(range(length)), sorted(results))

From 9c9e1a318d83b8e55125fdf7a7fb9482cb4951f8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Thu, 25 Mar 2021 11:58:39 -0300
Subject: [PATCH 331/716] [HTTP/1.1] Skip Content-Length header if its value is
 UNKNOWN_LENGTH (#5062)

---
 scrapy/core/downloader/handlers/http11.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 1f82751fd..25cb3ec62 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -361,10 +361,9 @@ class ScrapyAgent:
     @staticmethod
     def _headers_from_twisted_response(response):
         headers = Headers()
-        if response.length is not None:
+        if response.length != UNKNOWN_LENGTH:
             headers[b'Content-Length'] = str(response.length).encode()
-        for key, value in response.headers.getAllRawHeaders():
-            headers[key] = value
+        headers.update(response.headers.getAllRawHeaders())
         return headers
 
     def _cb_bodyready(self, txresponse, request):

From 0596f2bf6e7fc404333345d1cece01e31fabed8e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 23 Mar 2021 22:47:48 +0500
Subject: [PATCH 332/716] Remove not needed deferred_from_coro call.

---
 scrapy/core/scraper.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 3eae71af7..7eedbf33e 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -16,7 +16,6 @@ from scrapy.utils.defer import (
     aiter_errback,
     defer_fail,
     defer_succeed,
-    deferred_from_coro,
     iter_errback,
     parallel,
     parallel_async,
@@ -191,8 +190,8 @@ class Scraper:
             return defer_succeed(None)
         if isinstance(result, collections.abc.AsyncIterable):
             it = aiter_errback(result, self.handle_spider_error, request, response, spider)
-            dfd = deferred_from_coro(parallel_async(it, self.concurrent_items, self._process_spidermw_output,
-                                     request, response, spider))
+            dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,
+                                 request, response, spider)
         else:
             it = iter_errback(result, self.handle_spider_error, request, response, spider)
             dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,

From a97dc55c714d90378dbc8a819cae1a3a40056d6e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 25 Mar 2021 17:37:32 +0500
Subject: [PATCH 333/716] Add/improve docs.

---
 docs/topics/asyncio.rst           |  1 -
 docs/topics/coroutines.rst        | 13 ++++++++++++-
 docs/topics/spider-middleware.rst | 17 +++++++++++------
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 4addaa178..18712c928 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -38,7 +38,6 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the
 :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
 use it instead of the default asyncio event loop.
 
-
 .. _asyncio-await-dfd:
 
 Awaiting on Deferreds
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 279632653..9b50d9312 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -17,6 +17,10 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :class:`~scrapy.http.Request` callbacks.
 
+    .. versionchanged:: VERSION
+       Output of async callbacks is now processed asynchronously instead of collecting
+       all of it first.
+
 -   The :meth:`process_item` method of
     :ref:`item pipelines <topics-item-pipeline>`.
 
@@ -30,6 +34,13 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :ref:`Signal handlers that support deferreds <signal-deferred>`.
 
+-   The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+    method of :ref:`spider middlewares <custom-spider-middleware>`.
+
+    .. versionadded:: VERSION
+    .. note:: This method needs to be an async generator, not just a coroutine that
+              returns an iterable.
+
 Usage
 =====
 
@@ -76,7 +87,7 @@ This means you can use many useful Python libraries providing such code::
         async def parse_with_asyncio(self, response):
             async with aiohttp.ClientSession() as session:
                 async with session.get('https://additional.url') as additional_response:
-                    additional_data = await r.text()
+                    additional_data = await additional_response.text()
             # ... use response and additional_data to yield items and requests
 
 .. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index fc114a63f..d09693c16 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -98,20 +98,25 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
     .. method:: process_spider_output(response, result, spider)
 
+        .. versionchanged:: VERSION
+           Since VERSION this can take and return an :term:`python:asynchronous
+           iterable`.
+
         This method is called with the results returned from the Spider, after
         it has processed the response.
 
-        :meth:`process_spider_output` must return an iterable of
-        :class:`~scrapy.http.Request` objects and :ref:`item object
-        <topics-items>`.
+        :meth:`process_spider_output` must return an iterable (normal or
+        asynchronous) of :class:`~scrapy.http.Request` objects and 
+        :ref:`item objects <topics-items>`.
 
         :param response: the response which generated this output from the
           spider
         :type response: :class:`~scrapy.http.Response` object
 
         :param result: the result returned by the spider
-        :type result: an iterable of :class:`~scrapy.http.Request` objects and
-          :ref:`item object <topics-items>`
+        :type result: an iterable (normal or asynchronous) of
+          :class:`~scrapy.http.Request` objects and :ref:`item objects
+          <topics-items>`
 
         :param spider: the spider whose result is being processed
         :type spider: :class:`~scrapy.spiders.Spider` object
@@ -122,7 +127,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.http.Request` objects and :ref:`item object
+        iterable of :class:`~scrapy.http.Request` objects and :ref:`item objects
         <topics-items>`.
 
         If it returns ``None``, Scrapy will continue processing this exception,

From f422861ef49e8d0e0c2aa4de937fb77b95e063ca Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 25 Mar 2021 20:47:40 +0500
Subject: [PATCH 334/716] Add more tests for spider middlewares.

---
 tests/test_spidermiddleware.py | 188 +++++++++++++++++++++++++++++++++
 1 file changed, 188 insertions(+)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 78e926adc..2584dec21 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,11 +1,15 @@
+import collections.abc
 from unittest import mock
 
+from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
 from scrapy.spiders import Spider
 from scrapy.http import Request, Response
 from scrapy.exceptions import _InvalidOutput
+from scrapy.utils.asyncgen import _process_iterable_universal, as_async_generator, collect_asyncgen
+from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.test import get_crawler
 from scrapy.core.spidermw import SpiderMiddlewareManager
 
@@ -101,3 +105,187 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
         result = self._scrape_response()
         self.assertIsInstance(result, Failure)
         self.assertIsInstance(result.value, ZeroDivisionError)
+
+
+class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
+    """ Helpers for testing sync, async and mixed middlewares.
+
+    Should work for process_spider_output and, when it's supported, process_start_requests.
+    """
+
+    RESULT_COUNT = 3  # to simplify checks, let everything return 3 objects
+
+    @defer.inlineCallbacks
+    def _get_middleware_result(self, *mw_classes):
+        for mw_cls in mw_classes:
+            self.mwman._add_middleware(mw_cls())
+        result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
+        return result
+
+    def assertAsyncGeneratorNotIterable(self, o):
+        with self.assertRaisesRegex(TypeError,
+                                    "'(async_generator|MutableAsyncChain)' object is not iterable"):
+            list(o)
+
+    @defer.inlineCallbacks
+    def _test_simple_base(self, *mw_classes):
+        result = yield self._get_middleware_result(*mw_classes)
+        self.assertIsInstance(result, collections.abc.Iterable)
+        result_list = list(result)
+        self.assertEqual(len(result_list), self.RESULT_COUNT)
+        self.assertIsInstance(result_list[0], self.ITEM_TYPE)
+
+    @defer.inlineCallbacks
+    def _test_asyncgen_base(self, *mw_classes):
+        result = yield self._get_middleware_result(*mw_classes)
+        self.assertIsInstance(result, collections.abc.AsyncIterator)
+        result_list = yield deferred_from_coro(collect_asyncgen(result))
+        self.assertEqual(len(result_list), self.RESULT_COUNT)
+        self.assertIsInstance(result_list[0], self.ITEM_TYPE)
+
+    @defer.inlineCallbacks
+    def _test_asyncgen_fail(self, *mw_classes):
+        result = yield self._get_middleware_result(*mw_classes)
+        self.assertIsInstance(result, collections.abc.Iterable)
+        self.assertAsyncGeneratorNotIterable(result)
+
+
+class ProcessSpiderOutputSimpleMiddleware:
+    def process_spider_output(self, response, result, spider):
+        for r in result:
+            yield r
+
+
+class ProcessSpiderOutputAsyncGenMiddleware:
+    async def process_spider_output(self, response, result, spider):
+        async for r in as_async_generator(result):
+            yield r
+
+
+class ProcessSpiderOutputUniversalMiddleware:
+    def process_spider_output(self, response, result, spider):
+        @_process_iterable_universal
+        async def process(result):
+            async for r in result:
+                yield r
+        return process(result)
+
+
+class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
+    """ process_spider_output tests for simple callbacks"""
+
+    ITEM_TYPE = dict
+    MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
+    MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
+    MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
+
+    def _scrape_func(self, *args, **kwargs):
+        yield {'foo': 1}
+        yield {'foo': 2}
+        yield {'foo': 3}
+
+    def test_simple(self):
+        """ Simple mw """
+        return self._test_simple_base(self.MW_SIMPLE)
+
+    def test_asyncgen(self):
+        """ Asyncgen mw """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN)
+
+    def test_simple_asyncgen(self):
+        """ Simple mw -> asyncgen mw """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_SIMPLE)
+
+    def test_asyncgen_simple(self):
+        """ Asyncgen mw -> simple mw; cannot work """
+        return self._test_asyncgen_fail(self.MW_SIMPLE,
+                                        self.MW_ASYNCGEN)
+
+    def test_universal(self):
+        """ Universal mw """
+        return self._test_simple_base(self.MW_UNIVERSAL)
+
+    def test_universal_simple(self):
+        """ Universal mw -> simple mw """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      self.MW_UNIVERSAL)
+
+    def test_simple_universal(self):
+        """ Simple mw -> universal mw """
+        return self._test_simple_base(self.MW_UNIVERSAL,
+                                      self.MW_SIMPLE)
+
+    def test_universal_asyncgen(self):
+        """ Universal mw -> asyncgen mw """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_UNIVERSAL)
+
+    def test_asyncgen_universal(self):
+        """ Asyncgen mw -> universal mw """
+        return self._test_asyncgen_base(self.MW_UNIVERSAL,
+                                        self.MW_ASYNCGEN)
+
+
+class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
+    """ process_spider_output tests for async generator callbacks """
+
+    async def _scrape_func(self, *args, **kwargs):
+        for item in super()._scrape_func():
+            yield item
+
+    def test_simple(self):
+        """ Simple mw; cannot work """
+        return self._test_asyncgen_fail(self.MW_SIMPLE)
+
+    @defer.inlineCallbacks
+    def test_simple_asyncgen(self):
+        """ Simple mw -> asyncgen mw; cannot work """
+        result = yield self._get_middleware_result(
+            self.MW_ASYNCGEN,
+            self.MW_SIMPLE)
+        self.assertIsInstance(result, collections.abc.AsyncIterable)
+        self.assertAsyncGeneratorNotIterable(result)
+
+    def test_universal(self):
+        """ Universal mw """
+        return self._test_asyncgen_base(self.MW_UNIVERSAL)
+
+    def test_universal_simple(self):
+        """ Universal mw -> simple mw; cannot work """
+        return self._test_asyncgen_fail(self.MW_SIMPLE,
+                                        self.MW_UNIVERSAL)
+
+    def test_simple_universal(self):
+        """ Simple mw -> universal mw; cannot work """
+        return self._test_asyncgen_fail(self.MW_UNIVERSAL,
+                                        self.MW_SIMPLE)
+
+
+class ProcessStartRequestsSimpleMiddleware:
+    def process_start_requests(self, start_requests, spider):
+        for r in start_requests:
+            yield r
+
+
+class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
+    """ process_start_requests tests for simple start_requests"""
+
+    ITEM_TYPE = Request
+    MW_SIMPLE = ProcessStartRequestsSimpleMiddleware
+
+    def _start_requests(self):
+        for i in range(3):
+            yield Request(f'https://example.com/{i}', dont_filter=True)
+
+    @defer.inlineCallbacks
+    def _get_middleware_result(self, *mw_classes):
+        for mw_cls in mw_classes:
+            self.mwman._add_middleware(mw_cls())
+        start_requests = iter(self._start_requests())
+        results = yield self.mwman.process_start_requests(start_requests, self.spider)
+        return results
+
+    def test_simple(self):
+        """ Simple mw """
+        self._test_simple_base(self.MW_SIMPLE)

From 0638d6f01c41f12311b21f06fee5c71f5d08569f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 25 Mar 2021 21:58:29 +0500
Subject: [PATCH 335/716] Fix handling middlewares that change sync iterables
 into async.

---
 scrapy/core/spidermw.py | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 33e215971..230332673 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,7 +3,7 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
-import inspect
+import collections.abc
 from itertools import islice
 
 from twisted.python.failure import Failure
@@ -96,11 +96,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
     def _process_spider_output(self, response, spider, result, start_index=0):
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
-        if inspect.isasyncgen(result):
-            iter_class = MutableAsyncChain
+        if isinstance(result, collections.abc.AsyncIterator):
+            recovered = MutableAsyncChain()
         else:
-            iter_class = MutableChain
-        recovered = iter_class()
+            recovered = MutableChain()
 
         method_list = islice(self.methods['process_spider_output'], start_index, None)
         for method_index, method in enumerate(method_list, start=start_index):
@@ -121,16 +120,23 @@ class SpiderMiddlewareManager(MiddlewareManager):
                        f"iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
 
-        return iter_class(result, recovered)
+        # check this again as the middlewares could change "result" from sync to async
+        if isinstance(result, collections.abc.AsyncIterator):
+            return MutableAsyncChain(result, recovered)
+        else:
+            return MutableChain(result, recovered)
 
     def _process_callback_output(self, response, spider, result):
-        if inspect.isasyncgen(result):
-            iter_class = MutableAsyncChain
+        if isinstance(result, collections.abc.AsyncIterator):
+            recovered = MutableAsyncChain()
         else:
-            iter_class = MutableChain
-        recovered = iter_class()
+            recovered = MutableChain()
         result = self._evaluate_iterable(response, spider, result, 0, recovered)
-        return iter_class(self._process_spider_output(response, spider, result), recovered)
+        result = self._process_spider_output(response, spider, result)
+        if isinstance(result, collections.abc.AsyncIterator):
+            return MutableAsyncChain(result, recovered)
+        else:
+            return MutableChain(result, recovered)
 
     def scrape_response(self, scrape_func, response, request, spider):
         def process_callback_output(result):

From 1d200258a527668186ea84f14e7c2b23b3200ab5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Fri, 26 Mar 2021 10:45:26 -0300
Subject: [PATCH 336/716] Adjust h2 version requirement (#5066)

---
 setup.py | 2 +-
 tox.ini  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 767c6f6bf..2b60a10af 100644
--- a/setup.py
+++ b/setup.py
@@ -31,7 +31,7 @@ install_requires = [
     'zope.interface>=4.1.3',
     'protego>=0.1.15',
     'itemadapter>=0.1.0',
-    'h2>=3.2.0',
+    'h2>=3.0,<4.0',
 ]
 extras_require = {}
 cpython_dependencies = [
diff --git a/tox.ini b/tox.ini
index 6907c8906..353977519 100644
--- a/tox.ini
+++ b/tox.ini
@@ -71,7 +71,7 @@ commands =
 deps =
     cryptography==2.0
     cssselect==0.9.1
-    h2==3.2.0
+    h2==3.0
     itemadapter==0.1.0
     parsel==1.5.0
     Protego==0.1.15

From b5f501df7bc3917af3d144bb6556a763f381cd7e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 26 Mar 2021 20:17:41 +0500
Subject: [PATCH 337/716] Remove some unneeded code from
 _AsyncCooperatorAdapter.

---
 scrapy/utils/defer.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 554edc38c..ca3d79fa6 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -136,8 +136,6 @@ class _AsyncCooperatorAdapter:
         self.anext_deferred = None
         result = self.callable(result, *self.callable_args, **self.callable_kwargs)
         d = self.waiting_deferreds.pop(0)
-        if d.called:
-            raise ValueError('Deferred in waiting_deferreds already called')
         if isinstance(result, defer.Deferred):
             result.chainDeferred(d)
         else:
@@ -152,8 +150,6 @@ class _AsyncCooperatorAdapter:
         failure.trap(StopAsyncIteration)
         self.finished = True
         for d in self.waiting_deferreds:
-            if d.called:
-                raise ValueError('Deferred in waiting_deferreds already called')
             d.callback(None)
 
     def _call_anext(self):
@@ -162,9 +158,6 @@ class _AsyncCooperatorAdapter:
         self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
-    def __iter__(self):
-        return self
-
     def __next__(self):
         # This puts a new Deferred into self.waiting_deferreds and returns it.
         # It also calls __anext__() if needed.

From 6803779eb7e408b275da62f670aba9deaf3eade9 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 26 Mar 2021 22:29:07 +0500
Subject: [PATCH 338/716] Add more tests for _process_iterable_universal.

---
 scrapy/utils/asyncgen.py     |  2 +-
 tests/test_utils_asyncgen.py | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 39c94ad8a..a79552f76 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -57,6 +57,6 @@ def _process_iterable_universal(process_async):
         if hasattr(iterable, '__iter__'):
             # convert process_async to process_sync
             return process_sync(iterable, *args, **kwargs)
-        raise ValueError(f"Wrong iterable type {type(iterable)}")
+        raise TypeError(f"Wrong iterable type {type(iterable)}")
 
     return process
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index 2f4181d3d..41993a934 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -2,6 +2,7 @@ from twisted.trial import unittest
 
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal
 from scrapy.utils.defer import deferred_f_from_coro_f
+from scrapy.utils.test import get_web_client_agent_req
 
 
 class AsyncgenUtilsTest(unittest.TestCase):
@@ -26,6 +27,13 @@ async def process_iterable(iterable):
         yield i * 2
 
 
+@_process_iterable_universal
+async def process_iterable_awaiting(iterable):
+    async for i in iterable:
+        yield i * 2
+        await get_web_client_agent_req('http://example.com')
+
+
 class ProcessIterableUniversalTest(unittest.TestCase):
 
     def test_normal(self):
@@ -38,3 +46,22 @@ class ProcessIterableUniversalTest(unittest.TestCase):
         iterable = as_async_generator([1, 2, 3])
         results = await collect_asyncgen(process_iterable(iterable))
         self.assertEqual(results, [2, 4, 6])
+
+    @deferred_f_from_coro_f
+    async def test_blocking(self):
+        iterable = [1, 2, 3]
+        with self.assertRaisesRegex(RuntimeError, "Synchronously-called function"):
+            list(process_iterable_awaiting(iterable))
+
+    def test_invalid_iterable(self):
+        with self.assertRaisesRegex(TypeError, "Wrong iterable type"):
+            process_iterable(None)
+
+    @deferred_f_from_coro_f
+    async def test_invalid_process(self):
+        @_process_iterable_universal
+        def process_iterable_invalid(iterable):
+            pass
+
+        with self.assertRaisesRegex(ValueError, "process_async returned wrong type"):
+            list(process_iterable_invalid([]))

From b247fa9982a390e1380c46e390069c6058d97921 Mon Sep 17 00:00:00 2001
From: Ricardo Amendoeira <ricardo.filipe.amendoeira+github@gmail.com>
Date: Mon, 29 Mar 2021 01:48:28 +0100
Subject: [PATCH 339/716] Include loading settings in `Running multiple spiders
 in the same process` section

The example in the documentation doesn't take into account the project settings
---
 docs/topics/practices.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index cf1de1bd1..db1ed362e 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -118,6 +118,7 @@ Here is an example that runs multiple spiders simultaneously:
 ::
 
     import scrapy
+    from scrapy.utils.project import get_project_settings
     from scrapy.crawler import CrawlerProcess
 
     class MySpider1(scrapy.Spider):
@@ -128,7 +129,8 @@ Here is an example that runs multiple spiders simultaneously:
         # Your second spider definition
         ...
 
-    process = CrawlerProcess()
+    settings = get_project_settings()
+    process = CrawlerProcess(settings)
     process.crawl(MySpider1)
     process.crawl(MySpider2)
     process.start() # the script will block here until all crawling jobs are finished

From 90fe494ba2ab66faa49eb73914a1eae3cebacd0a Mon Sep 17 00:00:00 2001
From: Veniamin Gvozdikov <g.veniamin@googlemail.com>
Date: Thu, 1 Apr 2021 11:11:28 +0300
Subject: [PATCH 340/716] Rebranding, updated GA code

---
 docs/_templates/layout.html | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html
index a6f6cbda8..18a5231ee 100644
--- a/docs/_templates/layout.html
+++ b/docs/_templates/layout.html
@@ -3,14 +3,9 @@
 {% block footer %}
 {{ super() }}
 <script type="text/javascript">
-!function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t){var e=document.createElement("script");e.type="text/javascript";e.async=!0;e.src=("https:"===document.location.protocol?"https://":"http://")+"cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(e,n)};analytics.SNIPPET_VERSION="3.1.0";
-analytics.load("8UDQfnf3cyFSTsM4YANnW5sXmgZVILbA");
-analytics.page();
-}}();
-
 analytics.ready(function () {
     ga('require', 'linker');
-    ga('linker:autoLink', ['scrapinghub.com', 'crawlera.com']);
+    ga('linker:autoLink', ['zyte.com']);
 });
 </script>
 {% endblock %}

From d458ccff3b2d6df94df1aa86eeb7d2505d62f2d6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 1 Apr 2021 12:27:35 -0300
Subject: [PATCH 341/716] Retry request: priority_adjust cannot be float
 (Request.priority is int)

---
 scrapy/downloadermiddlewares/retry.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 2721db7cf..5965a1c6c 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -41,7 +41,7 @@ def get_retry_request(
     spider: Spider,
     reason: Union[str, Exception] = 'unspecified',
     max_retry_times: Optional[int] = None,
-    priority_adjust: Union[int, float, None] = None,
+    priority_adjust: Optional[int] = None,
     logger: Logger = retry_logger,
     stats_base_key: str = 'retry',
 ):

From 5492972d8a874e9fb7780175e4f396bc8f39378c Mon Sep 17 00:00:00 2001
From: anay2103 <55763427+anay2103@users.noreply.github.com>
Date: Thu, 1 Apr 2021 20:30:48 +0300
Subject: [PATCH 342/716] added customized filtering examples in logging.rst
 (#4965)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* added customized filtering examples in logging.rst

* Update logging.rst

* Update docs/topics/logging.rst

Co-authored-by: Adrián Chaves <adrian@chaves.io>

* Update docs/topics/logging.rst

Co-authored-by: Adrián Chaves <adrian@chaves.io>

* Update docs/topics/logging.rst

Co-authored-by: Adrián Chaves <adrian@chaves.io>

* Update docs/topics/logging.rst

Co-authored-by: Adrián Chaves <adrian@chaves.io>

* Update logging.rst

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/logging.rst | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index c3445d40e..00806392a 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -242,6 +242,47 @@ e.g. in the spider's ``__init__`` method::
 If you run this spider again then INFO messages from
 ``scrapy.spidermiddlewares.httperror`` logger will be gone.
 
+You can also filter log records by :class:`~logging.LogRecord` data. For 
+example, you can filter log records by message content using a substring or
+a regular expression. Create a :class:`logging.Filter` subclass 
+and equip it with a regular expression pattern to
+filter out unwanted messages::
+
+    import logging
+    import re
+    
+    class ContentFilter(logging.Filter):
+        def filter(self, record):
+            match = re.search(r'\d{3} [Ee]rror, retrying', record.message)
+            if match:
+                return False
+                
+A project-level filter may be attached to the root 
+handler created by Scrapy, this is a wieldy way to 
+filter all loggers in different parts of the project
+(middlewares, spider, etc.)::
+
+    import logging
+    import scrapy
+
+    class MySpider(scrapy.Spider):
+        # ...
+        def __init__(self, *args, **kwargs):
+            for handler in logging.root.handlers:
+                handler.addFilter(ContentFilter())
+ 
+Alternatively, you may choose a specific logger 
+and hide it without affecting other loggers::
+
+    import logging
+    import scrapy
+    
+    class MySpider(scrapy.Spider):
+        # ...
+        def __init__(self, *args, **kwargs):
+            logger = logging.getLogger('my_logger')
+            logger.addFilter(ContentFilter())
+            
 scrapy.utils.log module
 =======================
 

From ad7456746961fb23beb3ebc051ee1ab0ed82a91e Mon Sep 17 00:00:00 2001
From: Kader DJEHAF <daek@online.fr>
Date: Thu, 1 Apr 2021 19:33:56 +0200
Subject: [PATCH 343/716] Fix argument type (int -> bool) (#4950)

* Fix warning: Expected type 'bool', got 'int' instead

* Update defer.py

* Fix warning: Expected type 'bool', got 'int' instead

Co-authored-by: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
---
 scrapy/http/request/form.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 7f267c800..ef2eb3ba6 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -71,7 +71,7 @@ def _urlencode(seq, enc):
     values = [(to_bytes(k, enc), to_bytes(v, enc))
               for k, vs in seq
               for v in (vs if is_listlike(vs) else [vs])]
-    return urlencode(values, doseq=1)
+    return urlencode(values, doseq=True)
 
 
 def _get_form(response, formname, formid, formnumber, formxpath):

From cc095aa8950975cf203b1d33724cc499043e3f17 Mon Sep 17 00:00:00 2001
From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com>
Date: Thu, 1 Apr 2021 23:09:33 +0530
Subject: [PATCH 344/716] Windows pip installation guide (#4736)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* added initial steps

* fixing link

* python3 -> python

* remaining steps

* steps updated

* Update docs/intro/install.rst

Co-authored-by: Adrián Chaves <adrian@chaves.io>

* added link to Visual Studio

* removed 'install V'

* Update docs/intro/install.rst

Co-authored-by: Adrián Chaves <adrian@chaves.io>

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/intro/install.rst | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index bf919ce25..8581dde0b 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -12,6 +12,7 @@ Supported Python versions
 Scrapy requires Python 3.6+, either the CPython implementation (default) or
 the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
 
+.. _intro-install-scrapy:
 
 Installing Scrapy
 =================
@@ -29,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with::
 
     pip install Scrapy
 
+We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
+to avoid conflicting with your system packages.
+
 Note that sometimes this may require solving compilation issues for some Scrapy
 dependencies depending on your operating system, so be sure to check the
 :ref:`intro-install-platform-notes`.
 
-We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
-to avoid conflicting with your system packages.
-
 For more detailed and platform specifics instructions, as well as
 troubleshooting information, read on.
 
@@ -117,6 +118,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
 
   conda install -c conda-forge scrapy
 
+To install Scrapy on Windows using ``pip``:
+
+.. warning::
+    This installation method requires “Microsoft Visual C++” for installing some 
+    Scrapy dependencies, which demands significantly more disk space than Anaconda.
+
+#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer.
+
+#. Run the Visual Studio Installer.
+
+#. Under the Workloads section, select **C++ build tools**.
+
+#. Check the installation details and make sure following packages are selected as optional components:
+
+    * **MSVC**  (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) )
+    
+    * **Windows SDK**  (e.g Windows 10 SDK (10.0.18362.0))
+
+#. Install the Visual Studio Build Tools.
+
+Now, you should be able to :ref:`install Scrapy <intro-install-scrapy>` using ``pip``.
 
 .. _intro-install-ubuntu:
 
@@ -268,4 +290,6 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
 .. _zsh: https://www.zsh.org/
 .. _Anaconda: https://docs.anaconda.com/anaconda/
 .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
+.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio
+.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/
 .. _conda-forge: https://conda-forge.org/

From 9e7cbc05ae10ab6c1335d92b4d41429ba8e2bf24 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Thu, 1 Apr 2021 15:22:51 -0300
Subject: [PATCH 345/716] Fix type for urlencode's doseq argument

---
 scrapy/commands/bench.py | 2 +-
 tests/spiders.py         | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 999c987ea..6bdf9eae0 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider):
 
     def start_requests(self):
         qargs = {'total': self.total, 'show': self.show}
-        url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}'
+        url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}'
         return [scrapy.Request(url, dont_filter=True)]
 
     def parse(self, response):
diff --git a/tests/spiders.py b/tests/spiders.py
index 7e579098a..5b45f897e 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -45,7 +45,7 @@ class FollowAllSpider(MetaSpider):
         self.urls_visited = []
         self.times = []
         qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
-        url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}")
+        url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
         self.start_urls = [url]
 
     def parse(self, response):
@@ -245,7 +245,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
 
         for s in range(100):
             qargs = {'total': 10, 'seed': s}
-            url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}")
+            url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
             yield Request(url, meta={'seed': s})
             if self.fail_yielding:
                 2 / 0

From 849472535ef9490e8cc2a62cc7f1835b2bc3eaba Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 2 Apr 2021 20:20:35 +0500
Subject: [PATCH 346/716] Update docs.

---
 docs/topics/coroutines.rst        | 65 +++++++++++++++++++++++++++++++
 docs/topics/spider-middleware.rst | 14 ++++---
 scrapy/utils/asyncgen.py          |  1 +
 3 files changed, 74 insertions(+), 6 deletions(-)

diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 9b50d9312..6a39dcb5e 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -108,3 +108,68 @@ Common use cases for asynchronous code include:
   :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
 
 .. _aio-libs: https://github.com/aio-libs
+
+.. _async-spider-middlewares:
+
+Asynchronous spider middlewares
+===============================
+
+.. versionadded:: VERSION
+.. note:: This currently applies to
+          :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`.
+
+Middleware methods discussed here can take and return async iterables. They can
+return the same type of iterable or they can take a normal one and return an
+async one. If such method needs to return an async iterable it must be an async
+generator, not just a coroutine that returns an iterable.
+
+.. autofunction:: scrapy.utils.asyncgen.as_async_generator
+
+In the simplest form that supports both sync and async input it can be written
+like this::
+
+    from scrapy.utils.asyncgen import as_async_generator
+
+    class ProcessSpiderOutputAsyncGenMiddleware:
+        async def process_spider_output(self, response, result, spider):
+            async for r in as_async_generator(result):
+                # ... do something with r
+                yield r
+
+If the middleware input (the callback result for ``process_spider_output``) is
+an async iterable, all middlewares that process it must support it. The
+built-in ones do, but the ones in your project and 3rd-party ones will need to
+be updated to support it, as the code that expects a normal iterable will break
+on an async one. If these middlewares receive an async iterable, they must
+return one as well. On the other hand, if they receive a normal iterable, they
+shouldn't break and ideally should return a normal iterable too. There can be
+several possible implementations of this.
+
+The simplest one, always converting normal iterables to async ones, is provided
+above. Because a result of a middleware method is passed to the same method of
+the next middleware, it's only possible to mix middlewares with synchronous and
+asynchronous implementations of the same method if all synchronous ones are
+called first (which isn't always possible).
+
+Another option is to make separate methods for normal and async iterables and
+choose one at run time::
+
+    from inspect import isasyncgen
+
+    class ProcessSpiderOutputAsyncGenMiddleware:
+        def _normal_process_spider_output(self, response, result, spider):
+            # ... do something with normal result
+
+        async def _async_process_spider_output(self, response, result, spider):
+            # ... do the same with async result
+
+        def process_spider_output(self, response, result, spider):
+            if isasyncgen(result):
+                return self._async_process_spider_output(self, response, result, spider)
+            else:
+                return self._normal_process_spider_output(self, response, result, spider)
+
+If you are writing a middleware that you intend to publish or to use in many
+projects, this is likely the best way to implement it. It may be possible to
+extract common code from both methods to reduce code duplication, as in the
+simplest case the only difference between them will be ``for`` vs ``async for``.
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index d09693c16..d87b89292 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -105,18 +105,20 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         This method is called with the results returned from the Spider, after
         it has processed the response.
 
-        :meth:`process_spider_output` must return an iterable (normal or
-        asynchronous) of :class:`~scrapy.http.Request` objects and 
-        :ref:`item objects <topics-items>`.
+        :meth:`process_spider_output` must return an iterable of
+        :class:`~scrapy.http.Request` objects and :ref:`item objects
+        <topics-items>`.
+
+        .. note:: When defined as a :ref:`coroutine <async>`, this method needs
+                  to be an async generator, not just return an iterable.
 
         :param response: the response which generated this output from the
           spider
         :type response: :class:`~scrapy.http.Response` object
 
         :param result: the result returned by the spider
-        :type result: an iterable (normal or asynchronous) of
-          :class:`~scrapy.http.Request` objects and :ref:`item objects
-          <topics-items>`
+        :type result: an iterable of :class:`~scrapy.http.Request` objects and
+          :ref:`item objects <topics-items>`
 
         :param spider: the spider whose result is being processed
         :type spider: :class:`~scrapy.spiders.Spider` object
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index a79552f76..6c0bb1d10 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -11,6 +11,7 @@ async def collect_asyncgen(result):
 
 
 async def as_async_generator(it):
+    """ Wraps an iterator (sync or async) into an async generator. """
     if isinstance(it, collections.abc.AsyncIterator):
         async for r in it:
             yield r

From 30ed7fa349214ad11b4d13c981cbd2fc63ddaf42 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 2 Apr 2021 22:20:56 +0500
Subject: [PATCH 347/716] Some cleanup, make sync middlewares fail earlier.

---
 scrapy/core/spidermw.py        | 15 +++++++--------
 tests/test_spidermiddleware.py | 18 ++++--------------
 2 files changed, 11 insertions(+), 22 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 230332673..b24ccf6ae 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -20,10 +20,6 @@ def _isiterable(possible_iterator):
     return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__')
 
 
-def _fname(f):
-    return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"
-
-
 class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
@@ -48,7 +44,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             try:
                 result = method(response=response, spider=spider)
                 if result is not None:
-                    msg = (f"Middleware {_fname(method)} must return None "
+                    msg = (f"Middleware {method.__qualname__} must return None "
                            f"or raise an exception, got {type(result)}")
                     raise _InvalidOutput(msg)
             except _InvalidOutput:
@@ -88,7 +84,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             elif result is None:
                 continue
             else:
-                msg = (f"Middleware {_fname(method)} must return None "
+                msg = (f"Middleware {method.__qualname__} must return None "
                        f"or an iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
         return _failure
@@ -96,7 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
     def _process_spider_output(self, response, spider, result, start_index=0):
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
-        if isinstance(result, collections.abc.AsyncIterator):
+        result_async = isinstance(result, collections.abc.AsyncIterator)
+        if result_async:
             recovered = MutableAsyncChain()
         else:
             recovered = MutableChain()
@@ -116,9 +113,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
             if _isiterable(result):
                 result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
             else:
-                msg = (f"Middleware {_fname(method)} must return an "
+                msg = (f"Middleware {method.__qualname__} must return an "
                        f"iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
+            if result_async and isinstance(result, collections.abc.Iterator):
+                raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")
 
         # check this again as the middlewares could change "result" from sync to async
         if isinstance(result, collections.abc.AsyncIterator):
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 2584dec21..0a6b96c0c 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -122,11 +122,6 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
         result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
         return result
 
-    def assertAsyncGeneratorNotIterable(self, o):
-        with self.assertRaisesRegex(TypeError,
-                                    "'(async_generator|MutableAsyncChain)' object is not iterable"):
-            list(o)
-
     @defer.inlineCallbacks
     def _test_simple_base(self, *mw_classes):
         result = yield self._get_middleware_result(*mw_classes)
@@ -145,9 +140,8 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
 
     @defer.inlineCallbacks
     def _test_asyncgen_fail(self, *mw_classes):
-        result = yield self._get_middleware_result(*mw_classes)
-        self.assertIsInstance(result, collections.abc.Iterable)
-        self.assertAsyncGeneratorNotIterable(result)
+        with self.assertRaisesRegex(TypeError, "Synchronous .+ called with an async iterable"):
+            yield self._get_middleware_result(*mw_classes)
 
 
 class ProcessSpiderOutputSimpleMiddleware:
@@ -238,14 +232,10 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
         """ Simple mw; cannot work """
         return self._test_asyncgen_fail(self.MW_SIMPLE)
 
-    @defer.inlineCallbacks
     def test_simple_asyncgen(self):
         """ Simple mw -> asyncgen mw; cannot work """
-        result = yield self._get_middleware_result(
-            self.MW_ASYNCGEN,
-            self.MW_SIMPLE)
-        self.assertIsInstance(result, collections.abc.AsyncIterable)
-        self.assertAsyncGeneratorNotIterable(result)
+        return self._test_asyncgen_fail(self.MW_ASYNCGEN,
+                                        self.MW_SIMPLE)
 
     def test_universal(self):
         """ Universal mw """

From 7bd1d888d49d238622007e659e54af76e82bf1c1 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 2 Apr 2021 23:06:29 +0500
Subject: [PATCH 348/716] More robust sync/async middleware mix checking.

---
 scrapy/core/spidermw.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index b24ccf6ae..d0d292007 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -92,8 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
     def _process_spider_output(self, response, spider, result, start_index=0):
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
-        result_async = isinstance(result, collections.abc.AsyncIterator)
-        if result_async:
+        last_result_async = isinstance(result, collections.abc.AsyncIterator)
+        if last_result_async:
             recovered = MutableAsyncChain()
         else:
             recovered = MutableChain()
@@ -116,11 +116,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 msg = (f"Middleware {method.__qualname__} must return an "
                        f"iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
-            if result_async and isinstance(result, collections.abc.Iterator):
+            if last_result_async and isinstance(result, collections.abc.Iterator):
                 raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")
+            last_result_async = isinstance(result, collections.abc.AsyncIterator)
 
-        # check this again as the middlewares could change "result" from sync to async
-        if isinstance(result, collections.abc.AsyncIterator):
+        if last_result_async:
             return MutableAsyncChain(result, recovered)
         else:
             return MutableChain(result, recovered)

From 9e3b868dd83a3ac93dfe04b53d5130145154f478 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 3 Apr 2021 17:04:09 +0500
Subject: [PATCH 349/716] Use __qualname__ in middleware handling.

---
 scrapy/core/downloader/middleware.py |  9 +++------
 scrapy/core/spidermw.py              | 10 +++-------
 2 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index b0e612e43..177f3f760 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -36,8 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = yield deferred_from_coro(method(request=request, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput(
-                        f"Middleware {method.__self__.__class__.__name__}"
-                        ".process_request must return None, Response or "
+                        f"Middleware {method.__qualname__} must return None, Response or "
                         f"Request, got {response.__class__.__name__}"
                     )
                 if response:
@@ -55,8 +54,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
                 if not isinstance(response, (Response, Request)):
                     raise _InvalidOutput(
-                        f"Middleware {method.__self__.__class__.__name__}"
-                        ".process_response must return Response or Request, "
+                        f"Middleware {method.__qualname__} must return Response or Request, "
                         f"got {type(response)}"
                     )
                 if isinstance(response, Request):
@@ -70,8 +68,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput(
-                        f"Middleware {method.__self__.__class__.__name__}"
-                        ".process_exception must return None, Response or "
+                        f"Middleware {method.__qualname__} must return None, Response or "
                         f"Request, got {type(response)}"
                     )
                 if response:
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 289292da7..e8733c4ad 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -18,10 +18,6 @@ def _isiterable(possible_iterator):
     return hasattr(possible_iterator, '__iter__')
 
 
-def _fname(f):
-    return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"
-
-
 class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
@@ -46,7 +42,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             try:
                 result = method(response=response, spider=spider)
                 if result is not None:
-                    msg = (f"Middleware {_fname(method)} must return None "
+                    msg = (f"Middleware {method.__qualname__} must return None "
                            f"or raise an exception, got {type(result)}")
                     raise _InvalidOutput(msg)
             except _InvalidOutput:
@@ -83,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             elif result is None:
                 continue
             else:
-                msg = (f"Middleware {_fname(method)} must return None "
+                msg = (f"Middleware {method.__qualname__} must return None "
                        f"or an iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
         return _failure
@@ -108,7 +104,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             if _isiterable(result):
                 result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
             else:
-                msg = (f"Middleware {_fname(method)} must return an "
+                msg = (f"Middleware {method.__qualname__} must return an "
                        f"iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
 

From a9e96f99077865bc2d844ecf0ae174db97ba6d8b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 3 Apr 2021 17:40:45 +0500
Subject: [PATCH 350/716] Add typing for middleware and coroutine related code.

---
 scrapy/core/downloader/middleware.py | 14 ++++++++-----
 scrapy/core/spidermw.py              | 31 ++++++++++++++++++----------
 scrapy/middleware.py                 | 11 +++++-----
 scrapy/utils/asyncgen.py             |  5 ++++-
 scrapy/utils/defer.py                | 23 ++++++++++++---------
 scrapy/utils/python.py               |  7 ++++---
 6 files changed, 56 insertions(+), 35 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index b0e612e43..441fc9fa6 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -3,8 +3,12 @@ Downloader Middleware manager
 
 See documentation in docs/topics/downloader-middleware.rst
 """
-from twisted.internet import defer
+from typing import Callable, Union
 
+from twisted.internet import defer
+from twisted.python.failure import Failure
+
+from scrapy import Spider
 from scrapy.exceptions import _InvalidOutput
 from scrapy.http import Request, Response
 from scrapy.middleware import MiddlewareManager
@@ -29,9 +33,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         if hasattr(mw, 'process_exception'):
             self.methods['process_exception'].appendleft(mw.process_exception)
 
-    def download(self, download_func, request, spider):
+    def download(self, download_func: Callable, request: Request, spider: Spider):
         @defer.inlineCallbacks
-        def process_request(request):
+        def process_request(request: Request):
             for method in self.methods['process_request']:
                 response = yield deferred_from_coro(method(request=request, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
@@ -45,7 +49,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             return (yield download_func(request=request, spider=spider))
 
         @defer.inlineCallbacks
-        def process_response(response):
+        def process_response(response: Union[Response, Request]):
             if response is None:
                 raise TypeError("Received None in process_response")
             elif isinstance(response, Request):
@@ -64,7 +68,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             return response
 
         @defer.inlineCallbacks
-        def process_exception(failure):
+        def process_exception(failure: Failure):
             exception = failure.value
             for method in self.methods['process_exception']:
                 response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 289292da7..b09adf8e2 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,25 +3,32 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
+from collections.abc import Iterable, AsyncIterable
 from itertools import islice
+from typing import Callable, Union, Any
 
 from twisted.python.failure import Failure
 
+from scrapy import Request, Spider
 from scrapy.exceptions import _InvalidOutput
+from scrapy.http import Response
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.python import MutableChain
 
 
-def _isiterable(possible_iterator):
-    return hasattr(possible_iterator, '__iter__')
+def _isiterable(o):
+    return isinstance(o, Iterable)
 
 
 def _fname(f):
     return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"
 
 
+ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
+
+
 class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
@@ -41,7 +48,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         process_spider_exception = getattr(mw, 'process_spider_exception', None)
         self.methods['process_spider_exception'].appendleft(process_spider_exception)
 
-    def _process_spider_input(self, scrape_func, response, request, spider):
+    def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider):
         for method in self.methods['process_spider_input']:
             try:
                 result = method(response=response, spider=spider)
@@ -55,7 +62,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 return scrape_func(Failure(), request, spider)
         return scrape_func(response, request, spider)
 
-    def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to):
+    def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,
+                           exception_processor_index: int, recover_to: MutableChain):
         try:
             for r in iterable:
                 yield r
@@ -66,7 +74,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 raise
             recover_to.extend(exception_result)
 
-    def _process_spider_exception(self, response, spider, _failure, start_index=0):
+    def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index=0):
         exception = _failure.value
         # don't handle _InvalidOutput exception
         if isinstance(exception, _InvalidOutput):
@@ -88,7 +96,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 raise _InvalidOutput(msg)
         return _failure
 
-    def _process_spider_output(self, response, spider, result, start_index=0):
+    def _process_spider_output(self, response: Response, spider: Spider,
+                               result: Iterable, start_index=0):
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
         recovered = MutableChain()
@@ -114,21 +123,21 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
         return MutableChain(result, recovered)
 
-    def _process_callback_output(self, response, spider, result):
+    def _process_callback_output(self, response: Response, spider: Spider, result: Iterable):
         recovered = MutableChain()
         result = self._evaluate_iterable(response, spider, result, 0, recovered)
         return MutableChain(self._process_spider_output(response, spider, result), recovered)
 
-    def scrape_response(self, scrape_func, response, request, spider):
-        def process_callback_output(result):
+    def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider):
+        def process_callback_output(result: Iterable):
             return self._process_callback_output(response, spider, result)
 
-        def process_spider_exception(_failure):
+        def process_spider_exception(_failure: Failure):
             return self._process_spider_exception(response, spider, _failure)
 
         dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
         dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
         return dfd
 
-    def process_start_requests(self, start_requests, spider):
+    def process_start_requests(self, start_requests, spider: Spider):
         return self._process_chain('process_start_requests', start_requests, spider)
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 5040378ea..c53cfb814 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,6 +1,7 @@
-from collections import defaultdict, deque
 import logging
 import pprint
+from collections import defaultdict, deque
+from typing import Callable
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import create_instance, load_object
@@ -16,7 +17,7 @@ class MiddlewareManager:
 
     def __init__(self, *middlewares):
         self.middlewares = middlewares
-        self.methods = defaultdict(deque)
+        self.methods: dict[str, deque[Callable]] = defaultdict(deque)
         for mw in middlewares:
             self._add_middleware(mw)
 
@@ -58,13 +59,13 @@ class MiddlewareManager:
         if hasattr(mw, 'close_spider'):
             self.methods['close_spider'].appendleft(mw.close_spider)
 
-    def _process_parallel(self, methodname, obj, *args):
+    def _process_parallel(self, methodname: str, obj, *args):
         return process_parallel(self.methods[methodname], obj, *args)
 
-    def _process_chain(self, methodname, obj, *args):
+    def _process_chain(self, methodname: str, obj, *args):
         return process_chain(self.methods[methodname], obj, *args)
 
-    def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args):
+    def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args):
         return process_chain_both(self.methods[cb_methodname],
                                   self.methods[eb_methodname], obj, *args)
 
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 7f697af5f..c290e376c 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,4 +1,7 @@
-async def collect_asyncgen(result):
+from collections.abc import AsyncIterable
+
+
+async def collect_asyncgen(result: AsyncIterable):
     results = []
     async for x in result:
         results.append(x)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 6db9cc117..c382a00f7 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -3,16 +3,19 @@ Helper functions for dealing with Twisted deferreds
 """
 import asyncio
 import inspect
+from collections.abc import Coroutine
 from functools import wraps
+from typing import Callable, Iterable, Any
 
 from twisted.internet import defer, task
 from twisted.python import failure
+from twisted.python.failure import Failure
 
 from scrapy.exceptions import IgnoreRequest
 from scrapy.utils.reactor import is_asyncio_reactor_installed
 
 
-def defer_fail(_failure):
+def defer_fail(_failure: Failure):
     """Same as twisted.internet.defer.fail but delay calling errback until
     next reactor loop
 
@@ -47,7 +50,7 @@ def defer_result(result):
         return defer_succeed(result)
 
 
-def mustbe_deferred(f, *args, **kw):
+def mustbe_deferred(f: Callable, *args, **kw):
     """Same as twisted.internet.defer.maybeDeferred, but delay calling
     callback/errback to next reactor loop
     """
@@ -64,7 +67,7 @@ def mustbe_deferred(f, *args, **kw):
         return defer_result(result)
 
 
-def parallel(iterable, count, callable, *args, **named):
+def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named):
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
 
@@ -75,7 +78,7 @@ def parallel(iterable, count, callable, *args, **named):
     return defer.DeferredList([coop.coiterate(work) for _ in range(count)])
 
 
-def process_chain(callbacks, input, *a, **kw):
+def process_chain(callbacks: Iterable[Callable], input, *a, **kw):
     """Return a Deferred built by chaining the given callbacks"""
     d = defer.Deferred()
     for x in callbacks:
@@ -84,7 +87,7 @@ def process_chain(callbacks, input, *a, **kw):
     return d
 
 
-def process_chain_both(callbacks, errbacks, input, *a, **kw):
+def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw):
     """Return a Deferred built by chaining the given callbacks and errbacks"""
     d = defer.Deferred()
     for cb, eb in zip(callbacks, errbacks):
@@ -100,7 +103,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):
     return d
 
 
-def process_parallel(callbacks, input, *a, **kw):
+def process_parallel(callbacks: Iterable[Callable], input, *a, **kw):
     """Return a Deferred with the output of all successful calls to the given
     callbacks
     """
@@ -110,7 +113,7 @@ def process_parallel(callbacks, input, *a, **kw):
     return d
 
 
-def iter_errback(iterable, errback, *a, **kw):
+def iter_errback(iterable: Iterable, errback: Callable, *a, **kw):
     """Wraps an iterable calling an errback if an error is caught while
     iterating it.
     """
@@ -124,7 +127,7 @@ def iter_errback(iterable, errback, *a, **kw):
             errback(failure.Failure(), *a, **kw)
 
 
-def deferred_from_coro(o):
+def deferred_from_coro(o) -> Any:
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, defer.Deferred):
         return o
@@ -139,7 +142,7 @@ def deferred_from_coro(o):
     return o
 
 
-def deferred_f_from_coro_f(coro_f):
+def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]):
     """ Converts a coroutine function into a function that returns a Deferred.
 
     The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it.
@@ -151,7 +154,7 @@ def deferred_f_from_coro_f(coro_f):
     return f
 
 
-def maybeDeferred_coro(f, *args, **kw):
+def maybeDeferred_coro(f: Callable, *args, **kw):
     """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """
     try:
         result = f(*args, **kw)
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 5703fd4c3..bcc12f24f 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -8,6 +8,7 @@ import re
 import sys
 import warnings
 import weakref
+from collections.abc import Iterable
 from functools import partial, wraps
 from itertools import chain
 
@@ -335,15 +336,15 @@ else:
         gc.collect()
 
 
-class MutableChain:
+class MutableChain(Iterable):
     """
     Thin wrapper around itertools.chain, allowing to add iterables "in-place"
     """
 
-    def __init__(self, *args):
+    def __init__(self, *args: Iterable):
         self.data = chain.from_iterable(args)
 
-    def extend(self, *iterables):
+    def extend(self, *iterables: Iterable):
         self.data = chain(self.data, chain.from_iterable(iterables))
 
     def __iter__(self):

From 414dd1119a7e15c612de7bd0fe560b6ca30b505f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 3 Apr 2021 17:54:55 +0500
Subject: [PATCH 351/716] Drop an unused import.

---
 scrapy/core/spidermw.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index b09adf8e2..9a5305376 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,7 +3,7 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
-from collections.abc import Iterable, AsyncIterable
+from collections.abc import Iterable
 from itertools import islice
 from typing import Callable, Union, Any
 

From 7dc857668f16e8c52ff44662aceb32f93ae3d80e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 4 Apr 2021 16:15:33 +0500
Subject: [PATCH 352/716] Also some typing for Scraper.

---
 scrapy/core/scraper.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 0d3e3450f..4a3eff888 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -3,12 +3,14 @@ extracts information from them"""
 
 import logging
 from collections import deque
+from collections.abc import Iterable
+from typing import Union
 
 from itemadapter import is_item
 from twisted.internet import defer
 from twisted.python.failure import Failure
 
-from scrapy import signals
+from scrapy import signals, Spider
 from scrapy.core.spidermw import SpiderMiddlewareManager
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy.http import Request, Response
@@ -120,7 +122,7 @@ class Scraper:
             response, request, deferred = slot.next_response_request_deferred()
             self._scrape(response, request, spider).chainDeferred(deferred)
 
-    def _scrape(self, result, request, spider):
+    def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider):
         """
         Handle the downloaded response or failure through the spider callback/errback
         """
@@ -131,7 +133,7 @@ class Scraper:
         dfd.addCallback(self.handle_spider_output, request, result, spider)
         return dfd
 
-    def _scrape2(self, result, request, spider):
+    def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider):
         """
         Handle the different cases of request's result been a Response or a Failure
         """
@@ -141,7 +143,7 @@ class Scraper:
             dfd = self.call_spider(result, request, spider)
             return dfd.addErrback(self._log_download_errors, result, request, spider)
 
-    def call_spider(self, result, request, spider):
+    def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider):
         if isinstance(result, Response):
             if getattr(result, "request", None) is None:
                 result.request = request
@@ -156,7 +158,7 @@ class Scraper:
             dfd.addErrback(request.errback)
         return dfd.addCallback(iterate_spider_output)
 
-    def handle_spider_error(self, _failure, request, response, spider):
+    def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider):
         exc = _failure.value
         if isinstance(exc, CloseSpider):
             self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
@@ -177,7 +179,7 @@ class Scraper:
             spider=spider
         )
 
-    def handle_spider_output(self, result, request, response, spider):
+    def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider):
         if not result:
             return defer_succeed(None)
         it = iter_errback(result, self.handle_spider_error, request, response, spider)

From e7d51886ef90f5b2d4fa13911381680ac192fc37 Mon Sep 17 00:00:00 2001
From: Mayank Singhal <17mayanksinghal@gmail.com>
Date: Tue, 6 Apr 2021 02:21:18 +0530
Subject: [PATCH 353/716] Find bash from PATH instead of /bin/bash

---
 docs/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/Makefile b/docs/Makefile
index ff68bf1ae..87d5d3047 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -8,7 +8,7 @@ PYTHON       = python
 SPHINXOPTS   =
 PAPER        =
 SOURCES      =
-SHELL        = /bin/bash
+SHELL        = /usr/bin/env bash
 
 ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \
                 -D latex_elements.papersize=$(PAPER) \

From a71d6ef29da8e3dfa906fbaa74ee1db64567f856 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 6 Apr 2021 16:09:07 +0200
Subject: [PATCH 354/716] 2.5.0 release notes (#5028)

Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
---
 docs/news.rst                    | 193 ++++++++++++++++++++++++++++++-
 docs/topics/request-response.rst |   4 +-
 docs/topics/settings.rst         |   5 +-
 docs/topics/signals.rst          |   2 +-
 tests/test_webclient.py          |   2 +-
 5 files changed, 200 insertions(+), 6 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index d9fe897ad..0ea412e75 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,190 @@
 Release notes
 =============
 
+.. _release-2.5.0:
+
+Scrapy 2.5.0 (2021-04-06)
+-------------------------
+
+Highlights:
+
+-   Official Python 3.9 support
+
+-   Experimental :ref:`HTTP/2 support <http2>`
+
+-   New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function
+    to retry requests from spider callbacks
+
+-   New :class:`~scrapy.signals.headers_received` signal that allows stopping
+    downloads early
+
+-   New :class:`Response.protocol <scrapy.http.Response.protocol>` attribute
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+-   Removed all code that :ref:`was deprecated in 1.7.0 <1.7-deprecations>` and
+    had not :ref:`already been removed in 2.4.0 <2.4-deprecation-removals>`.
+    (:issue:`4901`)
+
+-   Removed support for the ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment
+    variable, :ref:`deprecated in 1.8.0 <1.8-deprecations>`. (:issue:`4912`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+-   The :mod:`scrapy.utils.py36` module is now deprecated in favor of
+    :mod:`scrapy.utils.asyncgen`. (:issue:`4900`)
+
+
+New features
+~~~~~~~~~~~~
+
+-   Experimental :ref:`HTTP/2 support <http2>` through a new download handler
+    that can be assigned to the ``https`` protocol in the
+    :setting:`DOWNLOAD_HANDLERS` setting.
+    (:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`)
+
+-   The new :func:`scrapy.downloadermiddlewares.retry.get_retry_request`
+    function may be used from spider callbacks or middlewares to handle the
+    retrying of a request beyond the scenarios that
+    :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` supports.
+    (:issue:`3590`, :issue:`3685`, :issue:`4902`)
+
+-   The new :class:`~scrapy.signals.headers_received` signal gives early access
+    to response headers and allows :ref:`stopping downloads
+    <topics-stop-response-download>`.
+    (:issue:`1772`, :issue:`4897`)
+
+-   The new :attr:`Response.protocol <scrapy.http.Response.protocol>`
+    attribute gives access to the string that identifies the protocol used to
+    download a response. (:issue:`4878`)
+
+-   :ref:`Stats <topics-stats>` now include the following entries that indicate
+    the number of successes and failures in storing
+    :ref:`feeds <topics-feed-exports>`::
+
+        feedexport/success_count/<storage type>
+        feedexport/failed_count/<storage type>
+
+    Where ``<storage type>`` is the feed storage backend class name, such as
+    :class:`~scrapy.extensions.feedexport.FileFeedStorage` or
+    :class:`~scrapy.extensions.feedexport.FTPFeedStorage`.
+
+    (:issue:`3947`, :issue:`4850`)
+
+-   The :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` spider
+    middleware now logs ignored URLs with ``INFO`` :ref:`logging level
+    <levels>` instead of ``DEBUG``, and it now includes the following entry
+    into :ref:`stats <topics-stats>` to keep track of the number of ignored
+    URLs::
+
+        urllength/request_ignored_count
+
+    (:issue:`5036`)
+
+-   The
+    :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
+    downloader middleware now logs the number of decompressed responses and the
+    total count of resulting bytes::
+
+        httpcompression/response_bytes
+        httpcompression/response_count
+
+    (:issue:`4797`, :issue:`4799`)
+
+
+Bug fixes
+~~~~~~~~~
+
+-   Fixed installation on PyPy installing PyDispatcher in addition to
+    PyPyDispatcher, which could prevent Scrapy from working depending on which
+    package got imported. (:issue:`4710`, :issue:`4814`)
+
+-   When inspecting a callback to check if it is a generator that also returns
+    a value, an exception is no longer raised if the callback has a docstring
+    with lower indentation than the following code.
+    (:issue:`4477`, :issue:`4935`)
+
+-   The `Content-Length <https://tools.ietf.org/html/rfc2616#section-14.13>`_
+    header is no longer omitted from responses when using the default, HTTP/1.1
+    download handler (see :setting:`DOWNLOAD_HANDLERS`).
+    (:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`)
+
+-   Setting the :reqmeta:`handle_httpstatus_all` request meta key to ``False``
+    now has the same effect as not setting it at all, instead of having the
+    same effect as setting it to ``True``.
+    (:issue:`3851`, :issue:`4694`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+-   Added instructions to :ref:`install Scrapy in Windows using pip
+    <intro-install-windows>`.
+    (:issue:`4715`, :issue:`4736`)
+
+-   Logging documentation now includes :ref:`additional ways to filter logs
+    <topics-logging-advanced-customization>`.
+    (:issue:`4216`, :issue:`4257`, :issue:`4965`)
+
+-   Covered how to deal with long lists of allowed domains in the :ref:`FAQ
+    <faq>`. (:issue:`2263`, :issue:`3667`)
+
+-   Covered scrapy-bench_ in :ref:`benchmarking`.
+    (:issue:`4996`, :issue:`5016`)
+
+-   Clarified that one :ref:`extension <topics-extensions>` instance is created
+    per crawler.
+    (:issue:`5014`)
+
+-   Fixed some errors in examples.
+    (:issue:`4829`, :issue:`4830`, :issue:`4907`, :issue:`4909`,
+    :issue:`5008`)
+
+-   Fixed some external links, typos, and so on.
+    (:issue:`4892`, :issue:`4899`, :issue:`4936`, :issue:`4942`, :issue:`5005`,
+    :issue:`5063`)
+
+-   The :ref:`list of Request.meta keys <topics-request-meta>` is now sorted
+    alphabetically.
+    (:issue:`5061`, :issue:`5065`)
+
+-   Updated references to Scrapinghub, which is now called Zyte.
+    (:issue:`4973`, :issue:`5072`)
+
+-   Added a mention to contributors in the README. (:issue:`4956`)
+
+-   Reduced the top margin of lists. (:issue:`4974`)
+
+
+Quality Assurance
+~~~~~~~~~~~~~~~~~
+
+-   Made Python 3.9 support official (:issue:`4757`, :issue:`4759`)
+
+-   Extended typing hints (:issue:`4895`)
+
+-   Fixed deprecated uses of the Twisted API.
+    (:issue:`4940`, :issue:`4950`, :issue:`5073`)
+
+-   Made our tests run with the new pip resolver.
+    (:issue:`4710`, :issue:`4814`)
+
+-   Added tests to ensure that :ref:`coroutine support <coroutine-support>`
+    is tested. (:issue:`4987`)
+
+-   Migrated from Travis CI to GitHub Actions. (:issue:`4924`)
+
+-   Fixed CI issues.
+    (:issue:`4986`, :issue:`5020`, :issue:`5022`, :issue:`5027`, :issue:`5052`,
+    :issue:`5053`)
+
+-   Implemented code refactorings, style fixes and cleanups.
+    (:issue:`4911`, :issue:`4982`, :issue:`5001`, :issue:`5002`, :issue:`5076`)
+
+
 .. _release-2.4.1:
 
 Scrapy 2.4.1 (2020-11-17)
@@ -97,6 +281,8 @@ Backward-incompatible changes
     (:issue:`4717`, :issue:`4823`)
 
 
+.. _2.4-deprecation-removals:
+
 Deprecation removals
 ~~~~~~~~~~~~~~~~~~~~
 
@@ -1433,6 +1619,8 @@ Deprecation removals
 *   ``scrapy.xlib`` has been removed (:issue:`4015`)
 
 
+.. _1.8-deprecations:
+
 Deprecations
 ~~~~~~~~~~~~
 
@@ -1789,6 +1977,8 @@ The following deprecated settings have also been removed (:issue:`3578`):
 *   ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`)
 
 
+.. _1.7-deprecations:
+
 Deprecations
 ~~~~~~~~~~~~
 
@@ -4184,7 +4374,7 @@ API changes
 - ``url`` and ``body`` attributes of Request objects are now read-only (#230)
 - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
 - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
-- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
+- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
 - Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself.
 - Changes to Scrapy Manager (now called "Crawler"):
    - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
@@ -4331,6 +4521,7 @@ First release of Scrapy.
 .. _resource: https://docs.python.org/2/library/resource.html
 .. _robots.txt: https://www.robotstxt.org/
 .. _scrapely: https://github.com/scrapy/scrapely
+.. _scrapy-bench: https://github.com/scrapy/scrapy-bench
 .. _service_identity: https://service-identity.readthedocs.io/en/stable/
 .. _six: https://six.readthedocs.io/
 .. _tox: https://pypi.org/project/tox/
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index c0283df01..500781c05 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -703,7 +703,7 @@ Response objects
     .. versionadded:: 2.1.0
        The ``ip_address`` parameter.
 
-    .. versionadded:: VERSION
+    .. versionadded:: 2.5.0
        The ``protocol`` parameter.
 
     .. attribute:: Response.url
@@ -809,7 +809,7 @@ Response objects
 
     .. attribute:: Response.protocol
 
-        .. versionadded:: VERSION
+        .. versionadded:: 2.5.0
 
         The protocol that was used to download the response.
         For instance: "HTTP/1.0", "HTTP/1.1"
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 9dcee64eb..f5dca824f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -677,6 +677,8 @@ handler (without replacement), place this in your ``settings.py``::
         'ftp': None,
     }
 
+.. _http2:
+
 The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
 :setting:`DOWNLOAD_HANDLERS` as follows::
 
@@ -703,7 +705,8 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
 
     -   No support for `server pushes`_, which are ignored.
 
-    -   No support for the :signal:`bytes_received` signal.
+    -   No support for the :signal:`bytes_received` and
+        :signal:`headers_received` signals.
 
 .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2
 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 98cfa606c..3d838fb63 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -403,7 +403,7 @@ bytes_received
 headers_received
 ~~~~~~~~~~~~~~~~
 
-.. versionadded:: VERSION
+.. versionadded:: 2.5
 
 .. signal:: headers_received
 .. function:: headers_received(headers, request, spider)
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index f935a8689..6e4cb9b6e 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -418,7 +418,7 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
 
     def testPayloadDisabledCipher(self):
         if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"):
-            self.skipTest("This does work in PyPy with cryptography<=2.3.1")
+            self.skipTest("This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1")
         s = "0123456789" * 10
         settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'})
         client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)

From e63188cbf753d560e43d8489c821bd6eb9fe54e9 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 6 Apr 2021 19:13:32 +0500
Subject: [PATCH 355/716] =?UTF-8?q?Bump=20version:=202.4.1=20=E2=86=92=202?=
 =?UTF-8?q?.5.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 956c512cb..d9e4a2831 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.4.1
+current_version = 2.5.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 005119baa..437459cd9 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.4.1
+2.5.0

From 8603f9d7a5524b7709b4e8a8fc04a75a9a4f0ffe Mon Sep 17 00:00:00 2001
From: Ricardo Amendoeira <ricardo.filipe.amendoeira+github@gmail.com>
Date: Tue, 6 Apr 2021 20:23:07 +0100
Subject: [PATCH 356/716] Apply changes to other examples in the same section.

---
 docs/topics/practices.rst | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index db1ed362e..15ac520e2 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -118,8 +118,8 @@ Here is an example that runs multiple spiders simultaneously:
 ::
 
     import scrapy
-    from scrapy.utils.project import get_project_settings
     from scrapy.crawler import CrawlerProcess
+    from scrapy.utils.project import get_project_settings
 
     class MySpider1(scrapy.Spider):
         # Your first spider definition
@@ -143,6 +143,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
     from twisted.internet import reactor
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
+    from scrapy.utils.project import get_project_settings
 
     class MySpider1(scrapy.Spider):
         # Your first spider definition
@@ -153,7 +154,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
         ...
 
     configure_logging()
-    runner = CrawlerRunner()
+    settings = get_project_settings()
+    runner = CrawlerRunner(settings)
     runner.crawl(MySpider1)
     runner.crawl(MySpider2)
     d = runner.join()
@@ -168,6 +170,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
     from twisted.internet import reactor, defer
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
+    from scrapy.utils.project import get_project_settings
 
     class MySpider1(scrapy.Spider):
         # Your first spider definition
@@ -178,7 +181,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
         ...
 
     configure_logging()
-    runner = CrawlerRunner()
+    settings = get_project_settings()
+    runner = CrawlerRunner(settings)
 
     @defer.inlineCallbacks
     def crawl():

From 5a75b14a5fbbbd37c14aa7317761655ac7706b70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 7 Apr 2021 12:33:37 +0200
Subject: [PATCH 357/716] docs: require sphinx-rtd-theme>=0.5.2 and the latest
 pip to prevent installing breaking docutils>=0.17

---
 .readthedocs.yml      | 1 -
 docs/pip.txt          | 3 ---
 docs/requirements.txt | 2 +-
 3 files changed, 1 insertion(+), 5 deletions(-)
 delete mode 100644 docs/pip.txt

diff --git a/.readthedocs.yml b/.readthedocs.yml
index 2d781ae81..80a1cd036 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -12,6 +12,5 @@ python:
   # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
   version: 3.8  # Keep in sync with .github/workflows/checks.yml
   install:
-    - requirements: docs/pip.txt
     - requirements: docs/requirements.txt
     - path: .
diff --git a/docs/pip.txt b/docs/pip.txt
deleted file mode 100644
index 095e53a0d..000000000
--- a/docs/pip.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-# In pip 20.3-21.0, the default dependency resolver causes the build in
-# ReadTheDocs to fail due to memory exhaustion or timeout.
-pip<20.3
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 3d34b47da..a0930ba1e 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,4 +1,4 @@
 Sphinx>=3.0
 sphinx-hoverxref>=0.2b1
 sphinx-notfound-page>=0.4
-sphinx_rtd_theme>=0.4
+sphinx-rtd-theme>=0.5.2
\ No newline at end of file

From 91f81445524630843561c8a3c71b7fa0f081d783 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 20 Nov 2019 00:30:18 -0300
Subject: [PATCH 358/716] Remove deprecated Spider.make_requests_from_url
 method

---
 scrapy/spiders/__init__.py | 27 ++-------------------------
 tests/test_spider.py       | 33 ---------------------------------
 2 files changed, 2 insertions(+), 58 deletions(-)

diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index c13ba4b3c..d8248c606 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -4,14 +4,12 @@ Base class for Scrapy spiders
 See documentation in docs/topics/spiders.rst
 """
 import logging
-import warnings
 from typing import Optional
 
 from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
-from scrapy.utils.deprecate import method_is_overridden
 
 
 class Spider(object_ref):
@@ -57,34 +55,13 @@ class Spider(object_ref):
         crawler.signals.connect(self.close, signals.spider_closed)
 
     def start_requests(self):
-        cls = self.__class__
         if not self.start_urls and hasattr(self, 'start_url'):
             raise AttributeError(
                 "Crawling could not start: 'start_urls' not found "
                 "or empty (but found 'start_url' attribute instead, "
                 "did you miss an 's'?)")
-        if method_is_overridden(cls, Spider, 'make_requests_from_url'):
-            warnings.warn(
-                "Spider.make_requests_from_url method is deprecated; it "
-                "won't be called in future Scrapy releases. Please "
-                "override Spider.start_requests method instead "
-                f"(see {cls.__module__}.{cls.__name__}).",
-            )
-            for url in self.start_urls:
-                yield self.make_requests_from_url(url)
-        else:
-            for url in self.start_urls:
-                yield Request(url, dont_filter=True)
-
-    def make_requests_from_url(self, url):
-        """ This method is deprecated. """
-        warnings.warn(
-            "Spider.make_requests_from_url method is deprecated: "
-            "it will be removed and not be called by the default "
-            "Spider.start_requests method in future Scrapy releases. "
-            "Please override Spider.start_requests method instead."
-        )
-        return Request(url, dont_filter=True)
+        for url in self.start_urls:
+            yield Request(url, dont_filter=True)
 
     def _parse(self, response, **kwargs):
         return self.parse(response, **kwargs)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index d23543f6a..a7c3ee048 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -584,39 +584,6 @@ class DeprecationTest(unittest.TestCase):
         assert issubclass(CrawlSpider, Spider)
         assert isinstance(CrawlSpider(name='foo'), Spider)
 
-    def test_make_requests_from_url_deprecated(self):
-        class MySpider4(Spider):
-            name = 'spider1'
-            start_urls = ['http://example.com']
-
-        class MySpider5(Spider):
-            name = 'spider2'
-            start_urls = ['http://example.com']
-
-            def make_requests_from_url(self, url):
-                return Request(url + "/foo", dont_filter=True)
-
-        with warnings.catch_warnings(record=True) as w:
-            # spider without overridden make_requests_from_url method
-            # doesn't issue a warning
-            spider1 = MySpider4()
-            self.assertEqual(len(list(spider1.start_requests())), 1)
-            self.assertEqual(len(w), 0)
-
-            # spider without overridden make_requests_from_url method
-            # should issue a warning when called directly
-            request = spider1.make_requests_from_url("http://www.example.com")
-            self.assertTrue(isinstance(request, Request))
-            self.assertEqual(len(w), 1)
-
-            # spider with overridden make_requests_from_url issues a warning,
-            # but the method still works
-            spider2 = MySpider5()
-            requests = list(spider2.start_requests())
-            self.assertEqual(len(requests), 1)
-            self.assertEqual(requests[0].url, 'http://example.com/foo')
-            self.assertEqual(len(w), 2)
-
 
 class NoParseMethodSpiderTest(unittest.TestCase):
 

From b6f77806b0ec414a28cfcbac3fa2d928040548d6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 9 Apr 2021 12:19:30 -0300
Subject: [PATCH 359/716] Engine tests: fix item class spider, add minimal type
 hints

---
 setup.cfg            |  3 ---
 tests/test_engine.py | 24 +++++++++++-------------
 2 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 89b4ec57e..1fab6fe22 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -43,9 +43,6 @@ ignore_errors = True
 [mypy-tests.test_downloader_handlers]
 ignore_errors = True
 
-[mypy-tests.test_engine]
-ignore_errors = True
-
 [mypy-tests.test_exporters]
 ignore_errors = True
 
diff --git a/tests/test_engine.py b/tests/test_engine.py
index ef1204f94..c406d2577 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -56,7 +56,7 @@ class TestSpider(Spider):
     name_re = re.compile(r"<h1>(.*?)</h1>", re.M)
     price_re = re.compile(r">Price: \$(.*?)<", re.M)
 
-    item_cls = TestItem
+    item_cls: type = TestItem
 
     def parse(self, response):
         xlink = LinkExtractor()
@@ -66,15 +66,15 @@ class TestSpider(Spider):
                 yield Request(url=link.url, callback=self.parse_item)
 
     def parse_item(self, response):
-        item = self.item_cls()
+        adapter = ItemAdapter(self.item_cls())
         m = self.name_re.search(response.text)
         if m:
-            item['name'] = m.group(1)
-        item['url'] = response.url
+            adapter['name'] = m.group(1)
+        adapter['url'] = response.url
         m = self.price_re.search(response.text)
         if m:
-            item['price'] = m.group(1)
-        return item
+            adapter['price'] = m.group(1)
+        return adapter.item
 
 
 class TestDupeFilterSpider(TestSpider):
@@ -87,7 +87,7 @@ class DictItemsSpider(TestSpider):
 
 
 class AttrsItemsSpider(TestSpider):
-    item_class = AttrsItem
+    item_cls = AttrsItem
 
 
 try:
@@ -97,14 +97,12 @@ except ImportError:
 else:
     TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)])
 
-    class DataClassItemsSpider(DictItemsSpider):
+    class _dataclass_spider(DictItemsSpider):
         def parse_item(self, response):
             item = super().parse_item(response)
-            return TestDataClass(
-                name=item.get('name'),
-                url=item.get('url'),
-                price=item.get('price'),
-            )
+            return TestDataClass(**item)
+
+    DataClassItemsSpider = _dataclass_spider
 
 
 class ItemZeroDivisionErrorSpider(TestSpider):

From 4673f05ddec85f18a890e606a6b85282035e26be Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 9 Apr 2021 23:42:24 +0500
Subject: [PATCH 360/716] Cleanup of slot handling in Scraper.

---
 scrapy/core/scraper.py | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 0d3e3450f..4a3ddea10 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -82,28 +82,26 @@ class Scraper:
 
     def close_spider(self, spider):
         """Close a spider being scraped and release its resources"""
-        slot = self.slot
-        slot.closing = defer.Deferred()
-        slot.closing.addCallback(self.itemproc.close_spider)
-        self._check_if_closing(spider, slot)
-        return slot.closing
+        self.slot.closing = defer.Deferred()
+        self.slot.closing.addCallback(self.itemproc.close_spider)
+        self._check_if_closing(spider)
+        return self.slot.closing
 
     def is_idle(self):
         """Return True if there isn't any more spiders to process"""
         return not self.slot
 
-    def _check_if_closing(self, spider, slot):
-        if slot.closing and slot.is_idle():
-            slot.closing.callback(spider)
+    def _check_if_closing(self, spider):
+        if self.slot.closing and self.slot.is_idle():
+            self.slot.closing.callback(spider)
 
     def enqueue_scrape(self, response, request, spider):
-        slot = self.slot
-        dfd = slot.add_response_request(response, request)
+        dfd = self.slot.add_response_request(response, request)
 
         def finish_scraping(_):
-            slot.finish_response(response, request)
-            self._check_if_closing(spider, slot)
-            self._scrape_next(spider, slot)
+            self.slot.finish_response(response, request)
+            self._check_if_closing(spider)
+            self._scrape_next(spider)
             return _
 
         dfd.addBoth(finish_scraping)
@@ -112,12 +110,12 @@ class Scraper:
                                    {'request': request},
                                    exc_info=failure_to_exc_info(f),
                                    extra={'spider': spider}))
-        self._scrape_next(spider, slot)
+        self._scrape_next(spider)
         return dfd
 
-    def _scrape_next(self, spider, slot):
-        while slot.queue:
-            response, request, deferred = slot.next_response_request_deferred()
+    def _scrape_next(self, spider):
+        while self.slot.queue:
+            response, request, deferred = self.slot.next_response_request_deferred()
             self._scrape(response, request, spider).chainDeferred(deferred)
 
     def _scrape(self, result, request, spider):

From d8d1dc5b508832598591ae9d3d3694f77f70492d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 12 Apr 2021 10:43:02 -0300
Subject: [PATCH 361/716] Ignore typing warning in test

---
 tests/test_engine.py | 4 +---
 tox.ini              | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index c406d2577..b2d1d83c7 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -97,13 +97,11 @@ except ImportError:
 else:
     TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)])
 
-    class _dataclass_spider(DictItemsSpider):
+    class DataClassItemsSpider(DictItemsSpider):  # type: ignore[no-redef]
         def parse_item(self, response):
             item = super().parse_item(response)
             return TestDataClass(**item)
 
-    DataClassItemsSpider = _dataclass_spider
-
 
 class ItemZeroDivisionErrorSpider(TestSpider):
     custom_settings = {
diff --git a/tox.ini b/tox.ini
index 353977519..5b0606f8f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -37,7 +37,7 @@ basepython = python3
 deps =
     mypy==0.780
 commands =
-    mypy {posargs: scrapy tests}
+    mypy --show-error-codes {posargs: scrapy tests}
 
 [testenv:security]
 basepython = python3

From a4415e4e6fa0f9becb40968438a77f4ea262633a Mon Sep 17 00:00:00 2001
From: Mayank Singhal <17mayanksinghal@gmail.com>
Date: Tue, 13 Apr 2021 17:20:55 +0530
Subject: [PATCH 362/716] Add DataURI download handler in
 DOWNLOAD_HANDLERS_BASE documentation

---
 docs/topics/settings.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f5dca824f..1d5babcec 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -657,6 +657,7 @@ DOWNLOAD_HANDLERS_BASE
 Default::
 
     {
+        'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
         'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
         'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
         'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',

From 76fa2257ef0280fc82e123457c791254cc2f185e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 13 Apr 2021 20:01:18 +0500
Subject: [PATCH 363/716] Add typing also for return values, other small fixes.

---
 scrapy/core/spidermw.py | 33 ++++++++++++++++++---------------
 scrapy/middleware.py    | 24 ++++++++++++++----------
 scrapy/utils/defer.py   | 24 ++++++++++++------------
 3 files changed, 44 insertions(+), 37 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index dc0e58095..05df8c988 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,10 +3,10 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
-from collections.abc import Iterable
 from itertools import islice
-from typing import Callable, Union, Any
+from typing import Callable, Union, Any, Generator, Iterable
 
+from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure
 
 from scrapy import Request, Spider
@@ -18,13 +18,13 @@ from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.python import MutableChain
 
 
-def _isiterable(o):
-    return isinstance(o, Iterable)
-
-
 ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
 
 
+def _isiterable(o) -> bool:
+    return isinstance(o, Iterable)
+
+
 class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
@@ -44,7 +44,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
         process_spider_exception = getattr(mw, 'process_spider_exception', None)
         self.methods['process_spider_exception'].appendleft(process_spider_exception)
 
-    def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider):
+    def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
+                              spider: Spider) -> Any:
         for method in self.methods['process_spider_input']:
             try:
                 result = method(response=response, spider=spider)
@@ -59,7 +60,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         return scrape_func(response, request, spider)
 
     def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,
-                           exception_processor_index: int, recover_to: MutableChain):
+                           exception_processor_index: int, recover_to: MutableChain) -> Generator:
         try:
             for r in iterable:
                 yield r
@@ -70,7 +71,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 raise
             recover_to.extend(exception_result)
 
-    def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index=0):
+    def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure,
+                                  start_index: int = 0) -> Union[Failure, MutableChain]:
         exception = _failure.value
         # don't handle _InvalidOutput exception
         if isinstance(exception, _InvalidOutput):
@@ -93,7 +95,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         return _failure
 
     def _process_spider_output(self, response: Response, spider: Spider,
-                               result: Iterable, start_index=0):
+                               result: Iterable, start_index: int = 0) -> MutableChain:
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
         recovered = MutableChain()
@@ -119,21 +121,22 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
         return MutableChain(result, recovered)
 
-    def _process_callback_output(self, response: Response, spider: Spider, result: Iterable):
+    def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain:
         recovered = MutableChain()
         result = self._evaluate_iterable(response, spider, result, 0, recovered)
         return MutableChain(self._process_spider_output(response, spider, result), recovered)
 
-    def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider):
-        def process_callback_output(result: Iterable):
+    def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
+                        spider: Spider) -> Deferred:
+        def process_callback_output(result: Iterable) -> MutableChain:
             return self._process_callback_output(response, spider, result)
 
-        def process_spider_exception(_failure: Failure):
+        def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
             return self._process_spider_exception(response, spider, _failure)
 
         dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
         dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
         return dfd
 
-    def process_start_requests(self, start_requests, spider: Spider):
+    def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
         return self._process_chain('process_start_requests', start_requests, spider)
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index c53cfb814..3f8c1cbf5 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,9 +1,13 @@
 import logging
 import pprint
 from collections import defaultdict, deque
-from typing import Callable
+from typing import Callable, Dict, Deque
 
+from twisted.internet import defer
+
+from scrapy import Spider
 from scrapy.exceptions import NotConfigured
+from scrapy.settings import Settings
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
 
@@ -17,16 +21,16 @@ class MiddlewareManager:
 
     def __init__(self, *middlewares):
         self.middlewares = middlewares
-        self.methods: dict[str, deque[Callable]] = defaultdict(deque)
+        self.methods: Dict[str, Deque[Callable]] = defaultdict(deque)
         for mw in middlewares:
             self._add_middleware(mw)
 
     @classmethod
-    def _get_mwlist_from_settings(cls, settings):
+    def _get_mwlist_from_settings(cls, settings: Settings) -> list:
         raise NotImplementedError
 
     @classmethod
-    def from_settings(cls, settings, crawler=None):
+    def from_settings(cls, settings: Settings, crawler=None):
         mwlist = cls._get_mwlist_from_settings(settings)
         middlewares = []
         enabled = []
@@ -53,24 +57,24 @@ class MiddlewareManager:
     def from_crawler(cls, crawler):
         return cls.from_settings(crawler.settings, crawler)
 
-    def _add_middleware(self, mw):
+    def _add_middleware(self, mw) -> None:
         if hasattr(mw, 'open_spider'):
             self.methods['open_spider'].append(mw.open_spider)
         if hasattr(mw, 'close_spider'):
             self.methods['close_spider'].appendleft(mw.close_spider)
 
-    def _process_parallel(self, methodname: str, obj, *args):
+    def _process_parallel(self, methodname: str, obj, *args) -> defer.Deferred:
         return process_parallel(self.methods[methodname], obj, *args)
 
-    def _process_chain(self, methodname: str, obj, *args):
+    def _process_chain(self, methodname: str, obj, *args) -> defer.Deferred:
         return process_chain(self.methods[methodname], obj, *args)
 
-    def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args):
+    def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> defer.Deferred:
         return process_chain_both(self.methods[cb_methodname],
                                   self.methods[eb_methodname], obj, *args)
 
-    def open_spider(self, spider):
+    def open_spider(self, spider: Spider) -> defer.Deferred:
         return self._process_parallel('open_spider', spider)
 
-    def close_spider(self, spider):
+    def close_spider(self, spider: Spider) -> defer.Deferred:
         return self._process_parallel('close_spider', spider)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index c382a00f7..095eae94c 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -5,7 +5,7 @@ import asyncio
 import inspect
 from collections.abc import Coroutine
 from functools import wraps
-from typing import Callable, Iterable, Any
+from typing import Callable, Iterable, Any, Generator
 
 from twisted.internet import defer, task
 from twisted.python import failure
@@ -15,7 +15,7 @@ from scrapy.exceptions import IgnoreRequest
 from scrapy.utils.reactor import is_asyncio_reactor_installed
 
 
-def defer_fail(_failure: Failure):
+def defer_fail(_failure: Failure) -> defer.Deferred:
     """Same as twisted.internet.defer.fail but delay calling errback until
     next reactor loop
 
@@ -28,7 +28,7 @@ def defer_fail(_failure: Failure):
     return d
 
 
-def defer_succeed(result):
+def defer_succeed(result) -> defer.Deferred:
     """Same as twisted.internet.defer.succeed but delay calling callback until
     next reactor loop
 
@@ -41,7 +41,7 @@ def defer_succeed(result):
     return d
 
 
-def defer_result(result):
+def defer_result(result) -> defer.Deferred:
     if isinstance(result, defer.Deferred):
         return result
     elif isinstance(result, failure.Failure):
@@ -50,7 +50,7 @@ def defer_result(result):
         return defer_succeed(result)
 
 
-def mustbe_deferred(f: Callable, *args, **kw):
+def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred:
     """Same as twisted.internet.defer.maybeDeferred, but delay calling
     callback/errback to next reactor loop
     """
@@ -67,7 +67,7 @@ def mustbe_deferred(f: Callable, *args, **kw):
         return defer_result(result)
 
 
-def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named):
+def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> defer.DeferredList:
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
 
@@ -78,7 +78,7 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named)
     return defer.DeferredList([coop.coiterate(work) for _ in range(count)])
 
 
-def process_chain(callbacks: Iterable[Callable], input, *a, **kw):
+def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred:
     """Return a Deferred built by chaining the given callbacks"""
     d = defer.Deferred()
     for x in callbacks:
@@ -87,7 +87,7 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw):
     return d
 
 
-def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw):
+def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred:
     """Return a Deferred built by chaining the given callbacks and errbacks"""
     d = defer.Deferred()
     for cb, eb in zip(callbacks, errbacks):
@@ -103,7 +103,7 @@ def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callabl
     return d
 
 
-def process_parallel(callbacks: Iterable[Callable], input, *a, **kw):
+def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred:
     """Return a Deferred with the output of all successful calls to the given
     callbacks
     """
@@ -113,7 +113,7 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw):
     return d
 
 
-def iter_errback(iterable: Iterable, errback: Callable, *a, **kw):
+def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:
     """Wraps an iterable calling an errback if an error is caught while
     iterating it.
     """
@@ -142,7 +142,7 @@ def deferred_from_coro(o) -> Any:
     return o
 
 
-def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]):
+def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable:
     """ Converts a coroutine function into a function that returns a Deferred.
 
     The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it.
@@ -154,7 +154,7 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]):
     return f
 
 
-def maybeDeferred_coro(f: Callable, *args, **kw):
+def maybeDeferred_coro(f: Callable, *args, **kw) -> defer.Deferred:
     """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """
     try:
         result = f(*args, **kw)

From 335a25675278543d4c85123bbbed99f228b26416 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Tue, 13 Apr 2021 21:05:20 +0500
Subject: [PATCH 364/716] Update scrapy/core/spidermw.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/core/spidermw.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 05df8c988..7e58521ac 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -4,7 +4,7 @@ Spider Middleware manager
 See documentation in docs/topics/spider-middleware.rst
 """
 from itertools import islice
-from typing import Callable, Union, Any, Generator, Iterable
+from typing import Any, Callable, Generator, Iterable, Union
 
 from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure

From b0e75125749ec1dce14614468bf06cd8e8842649 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Tue, 13 Apr 2021 21:05:25 +0500
Subject: [PATCH 365/716] Update scrapy/middleware.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/middleware.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 3f8c1cbf5..09768b59d 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,7 +1,7 @@
 import logging
 import pprint
 from collections import defaultdict, deque
-from typing import Callable, Dict, Deque
+from typing import Callable, Deque, Dict
 
 from twisted.internet import defer
 

From a8de04c823f5e10d012aaa2dd94a4f5a9f70b119 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Tue, 13 Apr 2021 21:05:30 +0500
Subject: [PATCH 366/716] Update scrapy/utils/defer.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/utils/defer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 095eae94c..e1139b1d1 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -5,7 +5,7 @@ import asyncio
 import inspect
 from collections.abc import Coroutine
 from functools import wraps
-from typing import Callable, Iterable, Any, Generator
+from typing import Any, Callable, Generator, Iterable
 
 from twisted.internet import defer, task
 from twisted.python import failure

From cef0a8b3d653d847efe32dfc2850e5992b627408 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 13 Apr 2021 21:07:07 +0500
Subject: [PATCH 367/716] Import Deferred directly in scrapy/middleware.py.

---
 scrapy/middleware.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 09768b59d..bbec38086 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -3,7 +3,7 @@ import pprint
 from collections import defaultdict, deque
 from typing import Callable, Deque, Dict
 
-from twisted.internet import defer
+from twisted.internet.defer import Deferred
 
 from scrapy import Spider
 from scrapy.exceptions import NotConfigured
@@ -63,18 +63,18 @@ class MiddlewareManager:
         if hasattr(mw, 'close_spider'):
             self.methods['close_spider'].appendleft(mw.close_spider)
 
-    def _process_parallel(self, methodname: str, obj, *args) -> defer.Deferred:
+    def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
         return process_parallel(self.methods[methodname], obj, *args)
 
-    def _process_chain(self, methodname: str, obj, *args) -> defer.Deferred:
+    def _process_chain(self, methodname: str, obj, *args) -> Deferred:
         return process_chain(self.methods[methodname], obj, *args)
 
-    def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> defer.Deferred:
+    def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred:
         return process_chain_both(self.methods[cb_methodname],
                                   self.methods[eb_methodname], obj, *args)
 
-    def open_spider(self, spider: Spider) -> defer.Deferred:
+    def open_spider(self, spider: Spider) -> Deferred:
         return self._process_parallel('open_spider', spider)
 
-    def close_spider(self, spider: Spider) -> defer.Deferred:
+    def close_spider(self, spider: Spider) -> Deferred:
         return self._process_parallel('close_spider', spider)

From 08e4eaf97369ba6daa4b5d84e00fd4d36b78e00a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 13 Apr 2021 22:41:01 +0500
Subject: [PATCH 368/716] Import Deferred directly in scrapy/utils/defer.py.

---
 scrapy/utils/defer.py | 48 ++++++++++++++++++++++---------------------
 1 file changed, 25 insertions(+), 23 deletions(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index e1139b1d1..b317c12a3 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -7,7 +7,9 @@ from collections.abc import Coroutine
 from functools import wraps
 from typing import Any, Callable, Generator, Iterable
 
-from twisted.internet import defer, task
+from twisted.internet import defer
+from twisted.internet.defer import Deferred, DeferredList, ensureDeferred
+from twisted.internet.task import Cooperator
 from twisted.python import failure
 from twisted.python.failure import Failure
 
@@ -15,7 +17,7 @@ from scrapy.exceptions import IgnoreRequest
 from scrapy.utils.reactor import is_asyncio_reactor_installed
 
 
-def defer_fail(_failure: Failure) -> defer.Deferred:
+def defer_fail(_failure: Failure) -> Deferred:
     """Same as twisted.internet.defer.fail but delay calling errback until
     next reactor loop
 
@@ -23,12 +25,12 @@ def defer_fail(_failure: Failure) -> defer.Deferred:
     before attending pending delayed calls, so do not set delay to zero.
     """
     from twisted.internet import reactor
-    d = defer.Deferred()
+    d = Deferred()
     reactor.callLater(0.1, d.errback, _failure)
     return d
 
 
-def defer_succeed(result) -> defer.Deferred:
+def defer_succeed(result) -> Deferred:
     """Same as twisted.internet.defer.succeed but delay calling callback until
     next reactor loop
 
@@ -36,13 +38,13 @@ def defer_succeed(result) -> defer.Deferred:
     before attending pending delayed calls, so do not set delay to zero.
     """
     from twisted.internet import reactor
-    d = defer.Deferred()
+    d = Deferred()
     reactor.callLater(0.1, d.callback, result)
     return d
 
 
-def defer_result(result) -> defer.Deferred:
-    if isinstance(result, defer.Deferred):
+def defer_result(result) -> Deferred:
+    if isinstance(result, Deferred):
         return result
     elif isinstance(result, failure.Failure):
         return defer_fail(result)
@@ -50,7 +52,7 @@ def defer_result(result) -> defer.Deferred:
         return defer_succeed(result)
 
 
-def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred:
+def mustbe_deferred(f: Callable, *args, **kw) -> Deferred:
     """Same as twisted.internet.defer.maybeDeferred, but delay calling
     callback/errback to next reactor loop
     """
@@ -67,29 +69,29 @@ def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred:
         return defer_result(result)
 
 
-def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> defer.DeferredList:
+def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> DeferredList:
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
 
     Taken from: https://jcalderone.livejournal.com/24285.html
     """
-    coop = task.Cooperator()
+    coop = Cooperator()
     work = (callable(elem, *args, **named) for elem in iterable)
-    return defer.DeferredList([coop.coiterate(work) for _ in range(count)])
+    return DeferredList([coop.coiterate(work) for _ in range(count)])
 
 
-def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred:
+def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
     """Return a Deferred built by chaining the given callbacks"""
-    d = defer.Deferred()
+    d = Deferred()
     for x in callbacks:
         d.addCallback(x, *a, **kw)
     d.callback(input)
     return d
 
 
-def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred:
+def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
     """Return a Deferred built by chaining the given callbacks and errbacks"""
-    d = defer.Deferred()
+    d = Deferred()
     for cb, eb in zip(callbacks, errbacks):
         d.addCallbacks(
             callback=cb, errback=eb,
@@ -103,12 +105,12 @@ def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callabl
     return d
 
 
-def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred:
+def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
     """Return a Deferred with the output of all successful calls to the given
     callbacks
     """
     dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks]
-    d = defer.DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
+    d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
     d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)
     return d
 
@@ -129,16 +131,16 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:
 
 def deferred_from_coro(o) -> Any:
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
-    if isinstance(o, defer.Deferred):
+    if isinstance(o, Deferred):
         return o
     if asyncio.isfuture(o) or inspect.isawaitable(o):
         if not is_asyncio_reactor_installed():
             # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
             # that use asyncio, e.g. "await asyncio.sleep(1)"
-            return defer.ensureDeferred(o)
+            return ensureDeferred(o)
         else:
             # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
-            return defer.Deferred.fromFuture(asyncio.ensure_future(o))
+            return Deferred.fromFuture(asyncio.ensure_future(o))
     return o
 
 
@@ -154,14 +156,14 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable:
     return f
 
 
-def maybeDeferred_coro(f: Callable, *args, **kw) -> defer.Deferred:
+def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
     """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """
     try:
         result = f(*args, **kw)
     except:  # noqa: E722
-        return defer.fail(failure.Failure(captureVars=defer.Deferred.debug))
+        return defer.fail(failure.Failure(captureVars=Deferred.debug))
 
-    if isinstance(result, defer.Deferred):
+    if isinstance(result, Deferred):
         return result
     elif asyncio.isfuture(result) or inspect.isawaitable(result):
         return deferred_from_coro(result)

From 61197d3dba53cd67c41e5d23e1f0c11a864f539b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 14 Apr 2021 18:21:58 +0500
Subject: [PATCH 369/716] Add/update typing, cleanup iterator/iterable
 inconsistencies.

---
 scrapy/core/scraper.py   |  7 +++---
 scrapy/core/spidermw.py  | 36 +++++++++++++++-------------
 scrapy/utils/asyncgen.py | 19 +++++++--------
 scrapy/utils/defer.py    | 52 ++++++++++++++++++++++++----------------
 scrapy/utils/python.py   | 14 +++++------
 5 files changed, 71 insertions(+), 57 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 8ce57af2f..0630ce625 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -1,10 +1,8 @@
 """This module implements the Scraper component which parses responses and
 extracts information from them"""
-import collections
 import logging
 from collections import deque
-from collections.abc import Iterable
-from typing import Union
+from typing import AsyncGenerator, AsyncIterable, Generator, Iterable, Union
 
 from itemadapter import is_item
 from twisted.internet import defer
@@ -188,7 +186,8 @@ class Scraper:
     def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider):
         if not result:
             return defer_succeed(None)
-        if isinstance(result, collections.abc.AsyncIterable):
+        it: Union[Generator, AsyncGenerator]
+        if isinstance(result, AsyncIterable):
             it = aiter_errback(result, self.handle_spider_error, request, response, spider)
             dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,
                                  request, response, spider)
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 6fcbc7492..9dd6c462d 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,9 +3,8 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
-import collections.abc
 from itertools import islice
-from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable
+from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable, AsyncGenerator
 
 from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure
@@ -61,8 +60,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 return scrape_func(Failure(), request, spider)
         return scrape_func(response, request, spider)
 
-    def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,
-                           exception_processor_index: int, recover_to: MutableChain) -> Generator:
+    def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable],
+                           exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain]
+                           ) -> Union[Generator, AsyncGenerator]:
         @_process_iterable_universal
         async def _evaluate_async_iterable(iterable):
             try:
@@ -100,11 +100,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
         return _failure
 
     def _process_spider_output(self, response: Response, spider: Spider,
-                               result: Iterable, start_index: int = 0) -> MutableChain:
+                               result: Union[Iterable, AsyncIterable], start_index: int = 0
+                               ) -> Union[MutableChain, MutableAsyncChain]:
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
-        last_result_async = isinstance(result, collections.abc.AsyncIterator)
-        if last_result_async:
+        recovered: Union[MutableChain, MutableAsyncChain]
+        last_result_is_async = isinstance(result, AsyncIterable)
+        if last_result_is_async:
             recovered = MutableAsyncChain()
         else:
             recovered = MutableChain()
@@ -127,30 +129,32 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 msg = (f"Middleware {method.__qualname__} must return an "
                        f"iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
-            if last_result_async and isinstance(result, collections.abc.Iterator):
+            if last_result_is_async and isinstance(result, Iterable):
                 raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")
-            last_result_async = isinstance(result, collections.abc.AsyncIterator)
+            last_result_is_async = isinstance(result, AsyncIterable)
 
-        if last_result_async:
+        if last_result_is_async:
             return MutableAsyncChain(result, recovered)
         else:
-            return MutableChain(result, recovered)
+            return MutableChain(result, recovered)  # type: ignore[arg-type]
 
-    def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain:
-        if isinstance(result, collections.abc.AsyncIterator):
+    def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
+                                 ) -> Union[MutableChain, MutableAsyncChain]:
+        recovered: Union[MutableChain, MutableAsyncChain]
+        if isinstance(result, AsyncIterable):
             recovered = MutableAsyncChain()
         else:
             recovered = MutableChain()
         result = self._evaluate_iterable(response, spider, result, 0, recovered)
         result = self._process_spider_output(response, spider, result)
-        if isinstance(result, collections.abc.AsyncIterator):
+        if isinstance(result, AsyncIterable):
             return MutableAsyncChain(result, recovered)
         else:
-            return MutableChain(result, recovered)
+            return MutableChain(result, recovered)  # type: ignore[arg-type]
 
     def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
                         spider: Spider) -> Deferred:
-        def process_callback_output(result: Iterable) -> MutableChain:
+        def process_callback_output(result: Union[Iterable, AsyncIterable]) -> Union[MutableChain, MutableAsyncChain]:
             return self._process_callback_output(response, spider, result)
 
         def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 92118ddb9..ae9a79989 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,7 +1,6 @@
-import collections
 import functools
 import inspect
-from collections.abc import AsyncIterable
+from typing import AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union
 
 
 async def collect_asyncgen(result: AsyncIterable):
@@ -11,9 +10,9 @@ async def collect_asyncgen(result: AsyncIterable):
     return results
 
 
-async def as_async_generator(it):
-    """ Wraps an iterator (sync or async) into an async generator. """
-    if isinstance(it, collections.abc.AsyncIterator):
+async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
+    """ Wraps an iterable (sync or async) into an async generator. """
+    if isinstance(it, AsyncIterable):
         async for r in it:
             yield r
     else:
@@ -22,7 +21,7 @@ async def as_async_generator(it):
 
 
 # https://stackoverflow.com/a/66170760/113586
-def _process_iterable_universal(process_async):
+def _process_iterable_universal(process_async: Callable):
     """ Takes a function that takes an async iterable, args and kwargs. Returns
     a function that takes any iterable, args and kwargs.
 
@@ -33,7 +32,7 @@ def _process_iterable_universal(process_async):
     # If this stops working, all internal uses can be just replaced with manually-written
     # process_sync functions.
 
-    def process_sync(iterable, *args, **kwargs):
+    def process_sync(iterable: Iterable, *args, **kwargs) -> Generator:
         agen = process_async(as_async_generator(iterable), *args, **kwargs)
         if not inspect.isasyncgen(agen):
             raise ValueError(f"process_async returned wrong type {type(agen)}")
@@ -52,11 +51,11 @@ def _process_iterable_universal(process_async):
                           f"you can't use {_process_iterable_universal.__name__} with it.")
 
     @functools.wraps(process_async)
-    def process(iterable, *args, **kwargs):
-        if inspect.isasyncgen(iterable):
+    def process(iterable: Union[Iterable, AsyncIterable], *args, **kwargs) -> Union[Generator, AsyncGenerator]:
+        if isinstance(iterable, AsyncIterable):
             # call process_async directly
             return process_async(iterable, *args, **kwargs)
-        if hasattr(iterable, '__iter__'):
+        if isinstance(iterable, Iterable):
             # convert process_async to process_sync
             return process_sync(iterable, *args, **kwargs)
         raise TypeError(f"Wrong iterable type {type(iterable)}")
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index f81faf6dd..39c8a85e9 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -3,9 +3,21 @@ Helper functions for dealing with Twisted deferreds
 """
 import asyncio
 import inspect
-from collections.abc import Coroutine
+from asyncio import Future
 from functools import wraps
-from typing import Any, Callable, Generator, Iterable
+from typing import (
+    Any,
+    AsyncGenerator,
+    AsyncIterable,
+    Callable,
+    Coroutine,
+    Generator,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Union
+)
 
 from twisted.internet import defer
 from twisted.internet.defer import Deferred, DeferredList, ensureDeferred
@@ -80,8 +92,8 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named)
     return DeferredList([coop.coiterate(work) for _ in range(count)])
 
 
-class _AsyncCooperatorAdapter:
-    """ A class that wraps an async iterator into a normal iterator suitable
+class _AsyncCooperatorAdapter(Iterator):
+    """ A class that wraps an async iterable into a normal iterator suitable
     for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
     it calls the callable directly in the callback, instead of providing a more
     generic interface.
@@ -125,30 +137,30 @@ class _AsyncCooperatorAdapter:
     Cooperator/CooperativeTask and use it instead of this adapter to achieve the same
     goal.
     """
-    def __init__(self, aiterator, callable, *callable_args, **callable_kwargs):
-        self.aiterator = aiterator
+    def __init__(self, aiterable: AsyncIterable, callable: Callable, *callable_args, **callable_kwargs):
+        self.aiterator = aiterable.__aiter__()
         self.callable = callable
         self.callable_args = callable_args
         self.callable_kwargs = callable_kwargs
         self.finished = False
-        self.waiting_deferreds = []
-        self.anext_deferred = None
+        self.waiting_deferreds: List[Deferred] = []
+        self.anext_deferred: Optional[Deferred] = None
 
-    def _callback(self, result):
+    def _callback(self, result: Any) -> None:
         # This gets called when the result from aiterator.__anext__() is available.
         # It calls the callable on it and sends the result to the oldest waiting Deferred
         # (by chaining if the result is a Deferred too or by firing if not).
         self.anext_deferred = None
         result = self.callable(result, *self.callable_args, **self.callable_kwargs)
         d = self.waiting_deferreds.pop(0)
-        if isinstance(result, defer.Deferred):
+        if isinstance(result, Deferred):
             result.chainDeferred(d)
         else:
             d.callback(None)
         if self.waiting_deferreds:
             self._call_anext()
 
-    def _errback(self, failure):
+    def _errback(self, failure: Failure) -> None:
         # This gets called on any exceptions in aiterator.__anext__().
         # It handles StopAsyncIteration by stopping the iteration and reraises all others.
         self.anext_deferred = None
@@ -157,29 +169,29 @@ class _AsyncCooperatorAdapter:
         for d in self.waiting_deferreds:
             d.callback(None)
 
-    def _call_anext(self):
+    def _call_anext(self) -> None:
         # This starts waiting for the next result from aiterator.
         # If aiterator is exhausted, _errback will be called.
         self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
-    def __next__(self):
+    def __next__(self) -> Deferred:
         # This puts a new Deferred into self.waiting_deferreds and returns it.
         # It also calls __anext__() if needed.
         if self.finished:
             raise StopIteration
-        d = defer.Deferred()
+        d = Deferred()
         self.waiting_deferreds.append(d)
         if not self.anext_deferred:
             self._call_anext()
         return d
 
 
-def parallel_async(async_iterable, count, callable, *args, **named):
-    """ Like parallel but for async iterables """
+def parallel_async(async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named) -> DeferredList:
+    """ Like parallel but for async iterators """
     coop = Cooperator()
     work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
-    dl = defer.DeferredList([coop.coiterate(work) for _ in range(count)])
+    dl = DeferredList([coop.coiterate(work) for _ in range(count)])
     return dl
 
 
@@ -232,7 +244,7 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:
             errback(failure.Failure(), *a, **kw)
 
 
-async def aiter_errback(aiterable, errback, *a, **kw):
+async def aiter_errback(aiterable: AsyncIterable, errback: Callable, *a, **kw) -> AsyncGenerator:
     """Wraps an async iterable calling an errback if an error is caught while
     iterating it. Similar to scrapy.utils.defer.iter_errback()
     """
@@ -290,13 +302,13 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
         return defer.succeed(result)
 
 
-def deferred_to_future(d):
+def deferred_to_future(d: Deferred) -> Future:
     """ Wraps a Deferred into a Future. Requires the asyncio reactor.
     """
     return d.asFuture(asyncio.get_event_loop())
 
 
-def maybe_deferred_to_future(d):
+def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
     """ Converts a Deferred to something that can be awaited in a callback or other user coroutine.
 
     If the asyncio reactor is installed, coroutines are wrapped into Futures, and only Futures can be
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 8b823d174..d086347bc 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -8,9 +8,9 @@ import re
 import sys
 import warnings
 import weakref
-from collections.abc import Iterable
 from functools import partial, wraps
 from itertools import chain
+from typing import AsyncIterable, Iterable, Union
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.asyncgen import as_async_generator
@@ -345,7 +345,7 @@ class MutableChain(Iterable):
     def __init__(self, *args: Iterable):
         self.data = chain.from_iterable(args)
 
-    def extend(self, *iterables: Iterable):
+    def extend(self, *iterables: Iterable) -> None:
         self.data = chain(self.data, chain.from_iterable(iterables))
 
     def __iter__(self):
@@ -359,22 +359,22 @@ class MutableChain(Iterable):
         return self.__next__()
 
 
-async def _async_chain(*iterables):
+async def _async_chain(*iterables: Union[Iterable, AsyncIterable]):
     for it in iterables:
         async for o in as_async_generator(it):
             yield o
 
 
-class MutableAsyncChain:
+class MutableAsyncChain(AsyncIterable):
     """
     Similar to MutableChain but for async iterables
     """
 
-    def __init__(self, *args):
+    def __init__(self, *args: Union[Iterable, AsyncIterable]):
         self.data = _async_chain(*args)
 
-    def extend(self, *aiterables):
-        self.data = _async_chain(self.data, _async_chain(*aiterables))
+    def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
+        self.data = _async_chain(self.data, _async_chain(*iterables))
 
     def __aiter__(self):
         return self

From 9db01a483c729369b272235bfb6e1c66ff62d1f7 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Wed, 14 Apr 2021 19:02:38 +0500
Subject: [PATCH 370/716] Update scrapy/core/spidermw.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/core/spidermw.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 9dd6c462d..d1fedae07 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -4,7 +4,7 @@ Spider Middleware manager
 See documentation in docs/topics/spider-middleware.rst
 """
 from itertools import islice
-from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable, AsyncGenerator
+from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union
 
 from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure

From 77bff0db0a6bdfca15295444f0e0eda47b35e702 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 14 Apr 2021 19:10:13 +0500
Subject: [PATCH 371/716] Additional typing for scraper and a small code
 change.

---
 scrapy/core/scraper.py | 100 ++++++++++++++++++++++++-----------------
 1 file changed, 59 insertions(+), 41 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index c760a4155..adbf8ef3d 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -3,11 +3,10 @@ extracts information from them"""
 
 import logging
 from collections import deque
-from collections.abc import Iterable
-from typing import Union
+from typing import Union, Optional, Tuple, Set, Deque, Any, Iterable
 
 from itemadapter import is_item
-from twisted.internet import defer
+from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.python.failure import Failure
 
 from scrapy import signals, Spider
@@ -20,6 +19,9 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.spider import iterate_spider_output
 
 
+QUEUE_TUPLE = Tuple[Union[Response, Failure], Request, Deferred]
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -28,46 +30,46 @@ class Slot:
 
     MIN_RESPONSE_SIZE = 1024
 
-    def __init__(self, max_active_size=5000000):
+    def __init__(self, max_active_size: int = 5000000):
         self.max_active_size = max_active_size
-        self.queue = deque()
-        self.active = set()
-        self.active_size = 0
-        self.itemproc_size = 0
-        self.closing = None
+        self.queue: Deque[QUEUE_TUPLE] = deque()
+        self.active: Set[Request] = set()
+        self.active_size: int = 0
+        self.itemproc_size: int = 0
+        self.closing: Optional[Deferred] = None
 
-    def add_response_request(self, response, request):
-        deferred = defer.Deferred()
-        self.queue.append((response, request, deferred))
-        if isinstance(response, Response):
-            self.active_size += max(len(response.body), self.MIN_RESPONSE_SIZE)
+    def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred:
+        deferred = Deferred()
+        self.queue.append((result, request, deferred))
+        if isinstance(result, Response):
+            self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE)
         else:
             self.active_size += self.MIN_RESPONSE_SIZE
         return deferred
 
-    def next_response_request_deferred(self):
+    def next_response_request_deferred(self) -> QUEUE_TUPLE:
         response, request, deferred = self.queue.popleft()
         self.active.add(request)
         return response, request, deferred
 
-    def finish_response(self, response, request):
+    def finish_response(self, result: Union[Response, Failure], request: Request) -> None:
         self.active.remove(request)
-        if isinstance(response, Response):
-            self.active_size -= max(len(response.body), self.MIN_RESPONSE_SIZE)
+        if isinstance(result, Response):
+            self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE)
         else:
             self.active_size -= self.MIN_RESPONSE_SIZE
 
-    def is_idle(self):
+    def is_idle(self) -> bool:
         return not (self.queue or self.active)
 
-    def needs_backout(self):
+    def needs_backout(self) -> bool:
         return self.active_size > self.max_active_size
 
 
 class Scraper:
 
     def __init__(self, crawler):
-        self.slot = None
+        self.slot: Optional[Slot] = None
         self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
         itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])
         self.itemproc = itemproc_cls.from_crawler(crawler)
@@ -76,32 +78,37 @@ class Scraper:
         self.signals = crawler.signals
         self.logformatter = crawler.logformatter
 
-    @defer.inlineCallbacks
-    def open_spider(self, spider):
+    @inlineCallbacks
+    def open_spider(self, spider: Spider):
         """Open the given spider for scraping and allocate resources for it"""
         self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE'))
         yield self.itemproc.open_spider(spider)
 
-    def close_spider(self, spider):
+    def close_spider(self, spider: Spider) -> Deferred:
         """Close a spider being scraped and release its resources"""
-        self.slot.closing = defer.Deferred()
+        if self.slot is None:
+            raise RuntimeError("Scraper slot not assigned")
+        self.slot.closing = Deferred()
         self.slot.closing.addCallback(self.itemproc.close_spider)
         self._check_if_closing(spider)
         return self.slot.closing
 
-    def is_idle(self):
+    def is_idle(self) -> bool:
         """Return True if there isn't any more spiders to process"""
         return not self.slot
 
-    def _check_if_closing(self, spider):
+    def _check_if_closing(self, spider: Spider) -> None:
+        assert self.slot is not None  # typing
         if self.slot.closing and self.slot.is_idle():
             self.slot.closing.callback(spider)
 
-    def enqueue_scrape(self, response, request, spider):
-        dfd = self.slot.add_response_request(response, request)
+    def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
+        if self.slot is None:
+            raise RuntimeError("Scraper slot not assigned")
+        dfd = self.slot.add_response_request(result, request)
 
         def finish_scraping(_):
-            self.slot.finish_response(response, request)
+            self.slot.finish_response(result, request)
             self._check_if_closing(spider)
             self._scrape_next(spider)
             return _
@@ -115,12 +122,13 @@ class Scraper:
         self._scrape_next(spider)
         return dfd
 
-    def _scrape_next(self, spider):
+    def _scrape_next(self, spider: Spider) -> None:
+        assert self.slot is not None  # typing
         while self.slot.queue:
             response, request, deferred = self.slot.next_response_request_deferred()
             self._scrape(response, request, spider).chainDeferred(deferred)
 
-    def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider):
+    def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
         """
         Handle the downloaded response or failure through the spider callback/errback
         """
@@ -131,7 +139,7 @@ class Scraper:
         dfd.addCallback(self.handle_spider_output, request, result, spider)
         return dfd
 
-    def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider):
+    def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
         """
         Handle the different cases of request's result been a Response or a Failure
         """
@@ -141,7 +149,7 @@ class Scraper:
             dfd = self.call_spider(result, request, spider)
             return dfd.addErrback(self._log_download_errors, result, request, spider)
 
-    def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider):
+    def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
         if isinstance(result, Response):
             if getattr(result, "request", None) is None:
                 result.request = request
@@ -156,7 +164,7 @@ class Scraper:
             dfd.addErrback(request.errback)
         return dfd.addCallback(iterate_spider_output)
 
-    def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider):
+    def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None:
         exc = _failure.value
         if isinstance(exc, CloseSpider):
             self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
@@ -177,7 +185,7 @@ class Scraper:
             spider=spider
         )
 
-    def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider):
+    def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred:
         if not result:
             return defer_succeed(None)
         it = iter_errback(result, self.handle_spider_error, request, response, spider)
@@ -185,10 +193,12 @@ class Scraper:
                        request, response, spider)
         return dfd
 
-    def _process_spidermw_output(self, output, request, response, spider):
+    def _process_spidermw_output(self, output: Any, request: Request, response: Response,
+                                 spider: Spider) -> Optional[Deferred]:
         """Process each Request/Item (given in the output parameter) returned
         from the given spider
         """
+        assert self.slot is not None  # typing
         if isinstance(output, Request):
             self.crawler.engine.crawl(request=output, spider=spider)
         elif is_item(output):
@@ -205,12 +215,18 @@ class Scraper:
                 {'request': request, 'typename': typename},
                 extra={'spider': spider},
             )
+        return None
 
-    def _log_download_errors(self, spider_failure, download_failure, request, spider):
+    def _log_download_errors(self, spider_failure: Failure, download_failure: Failure, request: Request,
+                             spider: Spider) -> Union[Failure, None]:
         """Log and silence errors that come from the engine (typically download
-        errors that got propagated thru here)
+        errors that got propagated thru here).
+
+        spider_failure: the value passed into the errback of self.call_spider()
+        download_failure: the value passed into _scrape2() from
+        ExecutionEngine._handle_downloader_output() as "result"
         """
-        if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest):
+        if not download_failure.check(IgnoreRequest):
             if download_failure.frames:
                 logkws = self.logformatter.download_error(download_failure, request, spider)
                 logger.log(
@@ -230,10 +246,12 @@ class Scraper:
 
         if spider_failure is not download_failure:
             return spider_failure
+        return None
 
-    def _itemproc_finished(self, output, item, response, spider):
+    def _itemproc_finished(self, output: Any, item: Any, response: Response, spider: Spider) -> None:
         """ItemProcessor finished for the given ``item`` and returned ``output``
         """
+        assert self.slot is not None  # typing
         self.slot.itemproc_size -= 1
         if isinstance(output, Failure):
             ex = output.value

From 309a637f32b0f6196eb4f77f19152527dcf3e5e7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 14 Apr 2021 20:26:37 +0500
Subject: [PATCH 372/716] Small changes.

---
 scrapy/core/scraper.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index adbf8ef3d..96aa53686 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -3,7 +3,7 @@ extracts information from them"""
 
 import logging
 from collections import deque
-from typing import Union, Optional, Tuple, Set, Deque, Any, Iterable
+from typing import Any, Deque, Iterable, Optional, Set, Tuple, Union
 
 from itemadapter import is_item
 from twisted.internet.defer import Deferred, inlineCallbacks
@@ -19,7 +19,7 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.spider import iterate_spider_output
 
 
-QUEUE_TUPLE = Tuple[Union[Response, Failure], Request, Deferred]
+QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
 
 
 logger = logging.getLogger(__name__)
@@ -32,7 +32,7 @@ class Slot:
 
     def __init__(self, max_active_size: int = 5000000):
         self.max_active_size = max_active_size
-        self.queue: Deque[QUEUE_TUPLE] = deque()
+        self.queue: Deque[QueueTuple] = deque()
         self.active: Set[Request] = set()
         self.active_size: int = 0
         self.itemproc_size: int = 0
@@ -47,7 +47,7 @@ class Slot:
             self.active_size += self.MIN_RESPONSE_SIZE
         return deferred
 
-    def next_response_request_deferred(self) -> QUEUE_TUPLE:
+    def next_response_request_deferred(self) -> QueueTuple:
         response, request, deferred = self.queue.popleft()
         self.active.add(request)
         return response, request, deferred

From 7e23677b52b659b11471a63f3be9905a0bbaf995 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Tue, 20 Apr 2021 08:45:28 -0300
Subject: [PATCH 373/716] Engine: deprecations and type hints (#5090)

---
 docs/topics/telnetconsole.rst                |   3 +-
 scrapy/core/engine.py                        | 419 ++++++++++---------
 scrapy/core/scraper.py                       |   2 +-
 scrapy/downloadermiddlewares/robotstxt.py    |   2 +-
 scrapy/extensions/memusage.py                |   6 +-
 scrapy/pipelines/media.py                    |   2 +-
 scrapy/shell.py                              |   2 +-
 scrapy/utils/engine.py                       |   3 +-
 tests/test_downloadermiddleware_robotstxt.py |  12 +-
 tests/test_engine.py                         | 110 ++++-
 10 files changed, 336 insertions(+), 225 deletions(-)

diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index 9802a34a2..832829b75 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -110,11 +110,10 @@ using the telnet console::
     Execution engine status
 
     time()-engine.start_time                        : 8.62972998619
-    engine.has_capacity()                           : False
     len(engine.downloader.active)                   : 16
     engine.scraper.is_idle()                        : False
     engine.spider.name                              : followall
-    engine.spider_is_idle(engine.spider)            : False
+    engine.spider_is_idle()                         : False
     engine.slot.closing                             : False
     len(engine.slot.inprogress)                     : 16
     len(engine.slot.scheduler.dqs or [])            : 0
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 93bcdb49a..edfac87c6 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -1,51 +1,61 @@
 """
-This is the Scrapy engine which controls the Scheduler, Downloader and Spiders.
+This is the Scrapy engine which controls the Scheduler, Downloader and Spider.
 
 For more information see docs/topics/architecture.rst
 
 """
 import logging
+import warnings
 from time import time
+from typing import Callable, Iterable, Iterator, Optional, Set, Union
 
-from twisted.internet import defer, task
+from twisted.internet.defer import Deferred, inlineCallbacks, succeed
+from twisted.internet.task import LoopingCall
 from twisted.python.failure import Failure
 
 from scrapy import signals
 from scrapy.core.scraper import Scraper
-from scrapy.exceptions import DontCloseSpider
+from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning
 from scrapy.http import Response, Request
-from scrapy.utils.misc import load_object
-from scrapy.utils.reactor import CallLaterOnce
+from scrapy.spiders import Spider
 from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
+from scrapy.utils.misc import create_instance, load_object
+from scrapy.utils.reactor import CallLaterOnce
+
 
 logger = logging.getLogger(__name__)
 
 
 class Slot:
-
-    def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
-        self.closing = False
-        self.inprogress = set()  # requests in progress
-        self.start_requests = iter(start_requests)
+    def __init__(
+        self,
+        start_requests: Iterable,
+        close_if_idle: bool,
+        nextcall: CallLaterOnce,
+        scheduler,
+    ) -> None:
+        self.closing: Optional[Deferred] = None
+        self.inprogress: Set[Request] = set()
+        self.start_requests: Optional[Iterator] = iter(start_requests)
         self.close_if_idle = close_if_idle
         self.nextcall = nextcall
         self.scheduler = scheduler
-        self.heartbeat = task.LoopingCall(nextcall.schedule)
+        self.heartbeat = LoopingCall(nextcall.schedule)
 
-    def add_request(self, request):
+    def add_request(self, request: Request) -> None:
         self.inprogress.add(request)
 
-    def remove_request(self, request):
+    def remove_request(self, request: Request) -> None:
         self.inprogress.remove(request)
         self._maybe_fire_closing()
 
-    def close(self):
-        self.closing = defer.Deferred()
+    def close(self) -> Deferred:
+        self.closing = Deferred()
         self._maybe_fire_closing()
         return self.closing
 
-    def _maybe_fire_closing(self):
-        if self.closing and not self.inprogress:
+    def _maybe_fire_closing(self) -> None:
+        if self.closing is not None and not self.inprogress:
             if self.nextcall:
                 self.nextcall.cancel()
                 if self.heartbeat.running:
@@ -54,210 +64,224 @@ class Slot:
 
 
 class ExecutionEngine:
-
-    def __init__(self, crawler, spider_closed_callback):
+    def __init__(self, crawler, spider_closed_callback: Callable) -> None:
         self.crawler = crawler
         self.settings = crawler.settings
         self.signals = crawler.signals
         self.logformatter = crawler.logformatter
-        self.slot = None
-        self.spider = None
+        self.slot: Optional[Slot] = None
+        self.spider: Optional[Spider] = None
         self.running = False
         self.paused = False
-        self.scheduler_cls = load_object(self.settings['SCHEDULER'])
+        self.scheduler_cls = load_object(crawler.settings["SCHEDULER"])
         downloader_cls = load_object(self.settings['DOWNLOADER'])
         self.downloader = downloader_cls(crawler)
         self.scraper = Scraper(crawler)
         self._spider_closed_callback = spider_closed_callback
 
-    @defer.inlineCallbacks
-    def start(self):
-        """Start the execution engine"""
+    @inlineCallbacks
+    def start(self) -> Deferred:
         if self.running:
             raise RuntimeError("Engine already running")
         self.start_time = time()
         yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
         self.running = True
-        self._closewait = defer.Deferred()
+        self._closewait = Deferred()
         yield self._closewait
 
-    def stop(self):
-        """Stop the execution engine gracefully"""
+    def stop(self) -> Deferred:
+        """Gracefully stop the execution engine"""
+        @inlineCallbacks
+        def _finish_stopping_engine(_) -> Deferred:
+            yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
+            self._closewait.callback(None)
+
         if not self.running:
             raise RuntimeError("Engine not running")
+
         self.running = False
-        dfd = self._close_all_spiders()
-        return dfd.addBoth(lambda _: self._finish_stopping_engine())
+        dfd = self.close_spider(self.spider, reason="shutdown") if self.spider is not None else succeed(None)
+        return dfd.addBoth(_finish_stopping_engine)
 
-    def close(self):
-        """Close the execution engine gracefully.
-
-        If it has already been started, stop it. In all cases, close all spiders
-        and the downloader.
+    def close(self) -> Deferred:
+        """
+        Gracefully close the execution engine.
+        If it has already been started, stop it. In all cases, close the spider and the downloader.
         """
         if self.running:
-            # Will also close spiders and downloader
-            return self.stop()
-        elif self.open_spiders:
-            # Will also close downloader
-            return self._close_all_spiders()
-        else:
-            return defer.succeed(self.downloader.close())
+            return self.stop()  # will also close spider and downloader
+        if self.spider is not None:
+            return self.close_spider(self.spider, reason="shutdown")  # will also close downloader
+        return succeed(self.downloader.close())
 
-    def pause(self):
-        """Pause the execution engine"""
+    def pause(self) -> None:
         self.paused = True
 
-    def unpause(self):
-        """Resume the execution engine"""
+    def unpause(self) -> None:
         self.paused = False
 
-    def _next_request(self, spider):
-        slot = self.slot
-        if not slot:
-            return
+    def _next_request(self) -> None:
+        assert self.slot is not None  # typing
+        assert self.spider is not None  # typing
 
         if self.paused:
-            return
+            return None
 
-        while not self._needs_backout(spider):
-            if not self._next_request_from_scheduler(spider):
-                break
+        while not self._needs_backout() and self._next_request_from_scheduler() is not None:
+            pass
 
-        if slot.start_requests and not self._needs_backout(spider):
+        if self.slot.start_requests is not None and not self._needs_backout():
             try:
-                request = next(slot.start_requests)
+                request = next(self.slot.start_requests)
             except StopIteration:
-                slot.start_requests = None
+                self.slot.start_requests = None
             except Exception:
-                slot.start_requests = None
-                logger.error('Error while obtaining start requests',
-                             exc_info=True, extra={'spider': spider})
+                self.slot.start_requests = None
+                logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': self.spider})
             else:
-                self.crawl(request, spider)
+                self.crawl(request)
 
-        if self.spider_is_idle(spider) and slot.close_if_idle:
-            self._spider_idle(spider)
+        if self.spider_is_idle() and self.slot.close_if_idle:
+            self._spider_idle()
 
-    def _needs_backout(self, spider):
-        slot = self.slot
+    def _needs_backout(self) -> bool:
         return (
             not self.running
-            or slot.closing
+            or self.slot.closing  # type: ignore[union-attr]
             or self.downloader.needs_backout()
-            or self.scraper.slot.needs_backout()
+            or self.scraper.slot.needs_backout()  # type: ignore[union-attr]
         )
 
-    def _next_request_from_scheduler(self, spider):
-        slot = self.slot
-        request = slot.scheduler.next_request()
-        if not request:
-            return
-        d = self._download(request, spider)
-        d.addBoth(self._handle_downloader_output, request, spider)
+    def _next_request_from_scheduler(self) -> Optional[Deferred]:
+        assert self.slot is not None  # typing
+        assert self.spider is not None  # typing
+
+        request = self.slot.scheduler.next_request()
+        if request is None:
+            return None
+
+        d = self._download(request, self.spider)
+        d.addBoth(self._handle_downloader_output, request, self.spider)
         d.addErrback(lambda f: logger.info('Error while handling downloader output',
                                            exc_info=failure_to_exc_info(f),
-                                           extra={'spider': spider}))
-        d.addBoth(lambda _: slot.remove_request(request))
+                                           extra={'spider': self.spider}))
+        d.addBoth(lambda _: self.slot.remove_request(request))
         d.addErrback(lambda f: logger.info('Error while removing request from slot',
                                            exc_info=failure_to_exc_info(f),
-                                           extra={'spider': spider}))
-        d.addBoth(lambda _: slot.nextcall.schedule())
+                                           extra={'spider': self.spider}))
+        d.addBoth(lambda _: self.slot.nextcall.schedule())
         d.addErrback(lambda f: logger.info('Error while scheduling new request',
                                            exc_info=failure_to_exc_info(f),
-                                           extra={'spider': spider}))
+                                           extra={'spider': self.spider}))
         return d
 
-    def _handle_downloader_output(self, response, request, spider):
-        if not isinstance(response, (Request, Response, Failure)):
-            raise TypeError(
-                "Incorrect type: expected Request, Response or Failure, got "
-                f"{type(response)}: {response!r}"
-            )
+    def _handle_downloader_output(
+        self, result: Union[Request, Response, Failure], request: Request, spider: Spider
+    ) -> Optional[Deferred]:
+        if not isinstance(result, (Request, Response, Failure)):
+            raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}")
+
         # downloader middleware can return requests (for example, redirects)
-        if isinstance(response, Request):
-            self.crawl(response, spider)
-            return
-        # response is a Response or Failure
-        d = self.scraper.enqueue_scrape(response, request, spider)
-        d.addErrback(lambda f: logger.error('Error while enqueuing downloader output',
-                                            exc_info=failure_to_exc_info(f),
-                                            extra={'spider': spider}))
+        if isinstance(result, Request):
+            self.crawl(result)
+            return None
+
+        d = self.scraper.enqueue_scrape(result, request, spider)
+        d.addErrback(
+            lambda f: logger.error(
+                "Error while enqueuing downloader output",
+                exc_info=failure_to_exc_info(f),
+                extra={'spider': spider},
+            )
+        )
         return d
 
-    def spider_is_idle(self, spider):
-        if not self.scraper.slot.is_idle():
-            # scraper is not idle
+    def spider_is_idle(self, spider: Optional[Spider] = None) -> bool:
+        if spider is not None:
+            warnings.warn(
+                "Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+        if self.slot is None:
+            raise RuntimeError("Engine slot not assigned")
+        if not self.scraper.slot.is_idle():  # type: ignore[union-attr]
             return False
-
-        if self.downloader.active:
-            # downloader has pending requests
+        if self.downloader.active:  # downloader has pending requests
             return False
-
-        if self.slot.start_requests is not None:
-            # not all start requests are handled
+        if self.slot.start_requests is not None:  # not all start requests are handled
             return False
-
         if self.slot.scheduler.has_pending_requests():
-            # scheduler has pending requests
             return False
-
         return True
 
-    @property
-    def open_spiders(self):
-        return [self.spider] if self.spider else []
+    def crawl(self, request: Request, spider: Optional[Spider] = None) -> None:
+        """Inject the request into the spider <-> downloader pipeline"""
+        if spider is not None:
+            warnings.warn(
+                "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            if spider is not self.spider:
+                raise RuntimeError(f"The spider {spider.name!r} does not match the open spider")
+        if self.spider is None:
+            raise RuntimeError(f"No open spider to crawl: {request}")
+        self._schedule_request(request, self.spider)
+        self.slot.nextcall.schedule()  # type: ignore[union-attr]
 
-    def has_capacity(self):
-        """Does the engine have capacity to handle more spiders"""
-        return not bool(self.slot)
-
-    def crawl(self, request, spider):
-        if spider not in self.open_spiders:
-            raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}")
-        self.schedule(request, spider)
-        self.slot.nextcall.schedule()
-
-    def schedule(self, request, spider):
+    def _schedule_request(self, request: Request, spider: Spider) -> None:
         self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider)
-        if not self.slot.scheduler.enqueue_request(request):
+        if not self.slot.scheduler.enqueue_request(request):  # type: ignore[union-attr]
             self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider)
 
-    def download(self, request, spider):
-        d = self._download(request, spider)
-        d.addBoth(self._downloaded, self.slot, request, spider)
-        return d
+    def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
+        """Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
+        if spider is None:
+            spider = self.spider
+        else:
+            warnings.warn(
+                "Passing a 'spider' argument to ExecutionEngine.download is deprecated",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            if spider is not self.spider:
+                logger.warning("The spider '%s' does not match the open spider", spider.name)
+        if spider is None:
+            raise RuntimeError(f"No open spider to crawl: {request}")
+        return self._download(request, spider).addBoth(self._downloaded, request, spider)
 
-    def _downloaded(self, response, slot, request, spider):
-        slot.remove_request(request)
-        return self.download(response, spider) if isinstance(response, Request) else response
+    def _downloaded(
+        self, result: Union[Response, Request], request: Request, spider: Spider
+    ) -> Union[Deferred, Response]:
+        assert self.slot is not None  # typing
+        self.slot.remove_request(request)
+        return self.download(result, spider) if isinstance(result, Request) else result
 
-    def _download(self, request, spider):
-        slot = self.slot
-        slot.add_request(request)
+    def _download(self, request: Request, spider: Spider) -> Deferred:
+        assert self.slot is not None  # typing
 
-        def _on_success(response):
-            if not isinstance(response, (Response, Request)):
-                raise TypeError(
-                    "Incorrect type: expected Response or Request, got "
-                    f"{type(response)}: {response!r}"
-                )
-            if isinstance(response, Response):
-                if response.request is None:
-                    response.request = request
-                logkws = self.logformatter.crawled(response.request, response, spider)
+        self.slot.add_request(request)
+
+        def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
+            if not isinstance(result, (Response, Request)):
+                raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}")
+            if isinstance(result, Response):
+                if result.request is None:
+                    result.request = request
+                logkws = self.logformatter.crawled(result.request, result, spider)
                 if logkws is not None:
-                    logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+                    logger.log(*logformatter_adapter(logkws), extra={"spider": spider})
                 self.signals.send_catch_log(
                     signal=signals.response_received,
-                    response=response,
-                    request=response.request,
+                    response=result,
+                    request=result.request,
                     spider=spider,
                 )
-            return response
+            return result
 
         def _on_complete(_):
-            slot.nextcall.schedule()
+            self.slot.nextcall.schedule()
             return _
 
         dwld = self.downloader.fetch(request, spider)
@@ -265,58 +289,52 @@ class ExecutionEngine:
         dwld.addBoth(_on_complete)
         return dwld
 
-    @defer.inlineCallbacks
-    def open_spider(self, spider, start_requests=(), close_if_idle=True):
-        if not self.has_capacity():
+    @inlineCallbacks
+    def open_spider(self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True):
+        if self.slot is not None:
             raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
         logger.info("Spider opened", extra={'spider': spider})
-        nextcall = CallLaterOnce(self._next_request, spider)
-        scheduler = self.scheduler_cls.from_crawler(self.crawler)
+        nextcall = CallLaterOnce(self._next_request)
+        scheduler = create_instance(self.scheduler_cls, settings=None, crawler=self.crawler)
         start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider)
-        slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
-        self.slot = slot
+        self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
         self.spider = spider
         yield scheduler.open(spider)
         yield self.scraper.open_spider(spider)
         self.crawler.stats.open_spider(spider)
         yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
-        slot.nextcall.schedule()
-        slot.heartbeat.start(5)
+        self.slot.nextcall.schedule()
+        self.slot.heartbeat.start(5)
 
-    def _spider_idle(self, spider):
-        """Called when a spider gets idle. This function is called when there
-        are no remaining pages to download or schedule. It can be called
-        multiple times. If some extension raises a DontCloseSpider exception
-        (in the spider_idle signal handler) the spider is not closed until the
-        next loop and this function is guaranteed to be called (at least) once
-        again for this spider.
+    def _spider_idle(self) -> None:
         """
-        res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider)
+        Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule.
+        It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider
+        exception, the spider is not closed until the next loop and this function is guaranteed to be called
+        (at least) once again.
+        """
+        assert self.spider is not None  # typing
+        res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider)
         if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
-            return
+            return None
+        if self.spider_is_idle():
+            self.close_spider(self.spider, reason='finished')
 
-        if self.spider_is_idle(spider):
-            self.close_spider(spider, reason='finished')
-
-    def close_spider(self, spider, reason='cancelled'):
+    def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
         """Close (cancel) spider and clear all its outstanding requests"""
+        if self.slot is None:
+            raise RuntimeError("Engine slot not assigned")
 
-        slot = self.slot
-        if slot.closing:
-            return slot.closing
-        logger.info("Closing spider (%(reason)s)",
-                    {'reason': reason},
-                    extra={'spider': spider})
+        if self.slot.closing is not None:
+            return self.slot.closing
 
-        dfd = slot.close()
+        logger.info("Closing spider (%(reason)s)", {'reason': reason}, extra={'spider': spider})
 
-        def log_failure(msg):
-            def errback(failure):
-                logger.error(
-                    msg,
-                    exc_info=failure_to_exc_info(failure),
-                    extra={'spider': spider}
-                )
+        dfd = self.slot.close()
+
+        def log_failure(msg: str) -> Callable:
+            def errback(failure: Failure) -> None:
+                logger.error(msg, exc_info=failure_to_exc_info(failure), extra={'spider': spider})
             return errback
 
         dfd.addBoth(lambda _: self.downloader.close())
@@ -325,19 +343,18 @@ class ExecutionEngine:
         dfd.addBoth(lambda _: self.scraper.close_spider(spider))
         dfd.addErrback(log_failure('Scraper close failure'))
 
-        dfd.addBoth(lambda _: slot.scheduler.close(reason))
+        dfd.addBoth(lambda _: self.slot.scheduler.close(reason))
         dfd.addErrback(log_failure('Scheduler close failure'))
 
         dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(
-            signal=signals.spider_closed, spider=spider, reason=reason))
+            signal=signals.spider_closed, spider=spider, reason=reason,
+        ))
         dfd.addErrback(log_failure('Error while sending spider_close signal'))
 
         dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))
         dfd.addErrback(log_failure('Stats close failure'))
 
-        dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)",
-                                          {'reason': reason},
-                                          extra={'spider': spider}))
+        dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", {'reason': reason}, extra={'spider': spider}))
 
         dfd.addBoth(lambda _: setattr(self, 'slot', None))
         dfd.addErrback(log_failure('Error while unassigning slot'))
@@ -349,12 +366,26 @@ class ExecutionEngine:
 
         return dfd
 
-    def _close_all_spiders(self):
-        dfds = [self.close_spider(s, reason='shutdown') for s in self.open_spiders]
-        dlist = defer.DeferredList(dfds)
-        return dlist
+    @property
+    def open_spiders(self) -> list:
+        warnings.warn(
+            "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return [self.spider] if self.spider is not None else []
 
-    @defer.inlineCallbacks
-    def _finish_stopping_engine(self):
-        yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
-        self._closewait.callback(None)
+    def has_capacity(self) -> bool:
+        warnings.warn("ExecutionEngine.has_capacity is deprecated", ScrapyDeprecationWarning, stacklevel=2)
+        return not bool(self.slot)
+
+    def schedule(self, request: Request, spider: Spider) -> None:
+        warnings.warn(
+            "ExecutionEngine.schedule is deprecated, please use "
+            "ExecutionEngine.crawl or ExecutionEngine.download instead",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        if self.slot is None:
+            raise RuntimeError("Engine slot not assigned")
+        self._schedule_request(request, spider)
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 96aa53686..d6d6f64f9 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -200,7 +200,7 @@ class Scraper:
         """
         assert self.slot is not None  # typing
         if isinstance(output, Request):
-            self.crawler.engine.crawl(request=output, spider=spider)
+            self.crawler.engine.crawl(request=output)
         elif is_item(output):
             self.slot.itemproc_size += 1
             dfd = self.itemproc.process_item(output, spider)
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index d6da55535..e66bf177e 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -67,7 +67,7 @@ class RobotsTxtMiddleware:
                 priority=self.DOWNLOAD_PRIORITY,
                 meta={'dont_obey_robotstxt': True}
             )
-            dfd = self.crawler.engine.download(robotsreq, spider)
+            dfd = self.crawler.engine.download(robotsreq)
             dfd.addCallback(self._parse_robots, netloc, spider)
             dfd.addErrback(self._logerror, robotsreq, spider)
             dfd.addErrback(self._robots_error, netloc)
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 274cbdbfe..9de119a10 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -88,10 +88,8 @@ class MemoryUsage:
                 self._send_report(self.notify_mails, subj)
                 self.crawler.stats.set_value('memusage/limit_notified', 1)
 
-            open_spiders = self.crawler.engine.open_spiders
-            if open_spiders:
-                for spider in open_spiders:
-                    self.crawler.engine.close_spider(spider, 'memusage_exceeded')
+            if self.crawler.engine.spider is not None:
+                self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded')
             else:
                 self.crawler.stop()
 
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 0c2ee6856..d1bccf323 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -173,7 +173,7 @@ class MediaPipeline:
                 errback=self.media_failed, errbackArgs=(request, info))
         else:
             self._modify_media_request(request)
-            dfd = self.crawler.engine.download(request, info.spider)
+            dfd = self.crawler.engine.download(request)
             dfd.addCallbacks(
                 callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item},
                 errback=self.media_failed, errbackArgs=(request, info))
diff --git a/scrapy/shell.py b/scrapy/shell.py
index c370ccaff..f2dff2ae3 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -79,7 +79,7 @@ class Shell:
         spider = self._open_spider(request, spider)
         d = _request_deferred(request)
         d.addCallback(lambda x: (x, spider))
-        self.crawler.engine.crawl(request, spider)
+        self.crawler.engine.crawl(request)
         return d
 
     def _open_spider(self, request, spider):
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index 0c1cee1a0..8e3ec2c37 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -8,11 +8,10 @@ def get_engine_status(engine):
     """Return a report of the current engine status"""
     tests = [
         "time()-engine.start_time",
-        "engine.has_capacity()",
         "len(engine.downloader.active)",
         "engine.scraper.is_idle()",
         "engine.spider.name",
-        "engine.spider_is_idle(engine.spider)",
+        "engine.spider_is_idle()",
         "engine.slot.closing",
         "len(engine.slot.inprogress)",
         "len(engine.slot.scheduler.dqs or [])",
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 858138f81..1460d88eb 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -42,7 +42,7 @@ Disallow: /some/randome/page.html
 """.encode('utf-8')
         response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
 
-        def return_response(request, spider):
+        def return_response(request):
             deferred = Deferred()
             reactor.callFromThread(deferred.callback, response)
             return deferred
@@ -79,7 +79,7 @@ Disallow: /some/randome/page.html
         crawler.settings.set('ROBOTSTXT_OBEY', True)
         response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2')
 
-        def return_response(request, spider):
+        def return_response(request):
             deferred = Deferred()
             reactor.callFromThread(deferred.callback, response)
             return deferred
@@ -102,7 +102,7 @@ Disallow: /some/randome/page.html
         crawler.settings.set('ROBOTSTXT_OBEY', True)
         response = Response('http://site.local/robots.txt')
 
-        def return_response(request, spider):
+        def return_response(request):
             deferred = Deferred()
             reactor.callFromThread(deferred.callback, response)
             return deferred
@@ -122,7 +122,7 @@ Disallow: /some/randome/page.html
         self.crawler.settings.set('ROBOTSTXT_OBEY', True)
         err = error.DNSLookupError('Robotstxt address not found')
 
-        def return_failure(request, spider):
+        def return_failure(request):
             deferred = Deferred()
             reactor.callFromThread(deferred.errback, failure.Failure(err))
             return deferred
@@ -138,7 +138,7 @@ Disallow: /some/randome/page.html
         self.crawler.settings.set('ROBOTSTXT_OBEY', True)
         err = error.DNSLookupError('Robotstxt address not found')
 
-        def immediate_failure(request, spider):
+        def immediate_failure(request):
             deferred = Deferred()
             deferred.errback(failure.Failure(err))
             return deferred
@@ -150,7 +150,7 @@ Disallow: /some/randome/page.html
     def test_ignore_robotstxt_request(self):
         self.crawler.settings.set('ROBOTSTXT_OBEY', True)
 
-        def ignore_request(request, spider):
+        def ignore_request(request):
             deferred = Deferred()
             reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest()))
             return deferred
diff --git a/tests/test_engine.py b/tests/test_engine.py
index b2d1d83c7..c200ded90 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -13,6 +13,7 @@ module with the ``runserver`` argument::
 import os
 import re
 import sys
+import warnings
 from collections import defaultdict
 from urllib.parse import urlparse
 
@@ -25,6 +26,7 @@ from twisted.web import server, static, util
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.item import Item, Field
 from scrapy.linkextractors import LinkExtractor
@@ -382,22 +384,104 @@ class EngineTest(unittest.TestCase):
         yield e.close()
 
     @defer.inlineCallbacks
-    def test_close_spiders_downloader(self):
-        e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
-        yield e.open_spider(TestSpider(), [])
-        self.assertEqual(len(e.open_spiders), 1)
-        yield e.close()
-        self.assertEqual(len(e.open_spiders), 0)
-
-    @defer.inlineCallbacks
-    def test_close_engine_spiders_downloader(self):
+    def test_start_already_running_exception(self):
         e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
         yield e.open_spider(TestSpider(), [])
         e.start()
-        self.assertTrue(e.running)
-        yield e.close()
-        self.assertFalse(e.running)
-        self.assertEqual(len(e.open_spiders), 0)
+        yield self.assertFailure(e.start(), RuntimeError).addBoth(
+            lambda exc: self.assertEqual(str(exc), "Engine already running")
+        )
+        yield e.stop()
+
+    @defer.inlineCallbacks
+    def test_close_spiders_downloader(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
+            yield e.open_spider(TestSpider(), [])
+            self.assertEqual(len(e.open_spiders), 1)
+            yield e.close()
+            self.assertEqual(len(e.open_spiders), 0)
+            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
+            self.assertEqual(
+                str(warning_list[0].message),
+                "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
+            )
+
+    @defer.inlineCallbacks
+    def test_close_engine_spiders_downloader(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
+            yield e.open_spider(TestSpider(), [])
+            e.start()
+            self.assertTrue(e.running)
+            yield e.close()
+            self.assertFalse(e.running)
+            self.assertEqual(len(e.open_spiders), 0)
+            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
+            self.assertEqual(
+                str(warning_list[0].message),
+                "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
+            )
+
+    @defer.inlineCallbacks
+    def test_crawl_deprecated_spider_arg(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
+            spider = TestSpider()
+            yield e.open_spider(spider, [])
+            e.start()
+            e.crawl(Request("data:,"), spider)
+            yield e.close()
+            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
+            self.assertEqual(
+                str(warning_list[0].message),
+                "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated",
+            )
+
+    @defer.inlineCallbacks
+    def test_download_deprecated_spider_arg(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
+            spider = TestSpider()
+            yield e.open_spider(spider, [])
+            e.start()
+            e.download(Request("data:,"), spider)
+            yield e.close()
+            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
+            self.assertEqual(
+                str(warning_list[0].message),
+                "Passing a 'spider' argument to ExecutionEngine.download is deprecated",
+            )
+
+    @defer.inlineCallbacks
+    def test_deprecated_schedule(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
+            spider = TestSpider()
+            yield e.open_spider(spider, [])
+            e.start()
+            e.schedule(Request("data:,"), spider)
+            yield e.close()
+            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
+            self.assertEqual(
+                str(warning_list[0].message),
+                "ExecutionEngine.schedule is deprecated, please use "
+                "ExecutionEngine.crawl or ExecutionEngine.download instead",
+            )
+
+    @defer.inlineCallbacks
+    def test_deprecated_has_capacity(self):
+        with warnings.catch_warnings(record=True) as warning_list:
+            e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
+            self.assertTrue(e.has_capacity())
+            spider = TestSpider()
+            yield e.open_spider(spider, [])
+            self.assertFalse(e.has_capacity())
+            e.start()
+            yield e.close()
+            self.assertTrue(e.has_capacity())
+            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
+            self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated")
 
 
 if __name__ == "__main__":

From e3f81d8d5f17515b6eba135ac0db7e270ff0a9f0 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Tue, 20 Apr 2021 11:46:43 -0300
Subject: [PATCH 374/716] Engine: remove unnecessary parameter (#5106)

---
 scrapy/core/engine.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index edfac87c6..7a09bafa1 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -161,7 +161,7 @@ class ExecutionEngine:
             return None
 
         d = self._download(request, self.spider)
-        d.addBoth(self._handle_downloader_output, request, self.spider)
+        d.addBoth(self._handle_downloader_output, request)
         d.addErrback(lambda f: logger.info('Error while handling downloader output',
                                            exc_info=failure_to_exc_info(f),
                                            extra={'spider': self.spider}))
@@ -176,8 +176,10 @@ class ExecutionEngine:
         return d
 
     def _handle_downloader_output(
-        self, result: Union[Request, Response, Failure], request: Request, spider: Spider
+        self, result: Union[Request, Response, Failure], request: Request
     ) -> Optional[Deferred]:
+        assert self.spider is not None  # typing
+
         if not isinstance(result, (Request, Response, Failure)):
             raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}")
 
@@ -186,12 +188,12 @@ class ExecutionEngine:
             self.crawl(result)
             return None
 
-        d = self.scraper.enqueue_scrape(result, request, spider)
+        d = self.scraper.enqueue_scrape(result, request, self.spider)
         d.addErrback(
             lambda f: logger.error(
                 "Error while enqueuing downloader output",
                 exc_info=failure_to_exc_info(f),
-                extra={'spider': spider},
+                extra={'spider': self.spider},
             )
         )
         return d

From e779ed7d93beec36f565d33a1cc8d3e8fe6068d7 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Tue, 20 Apr 2021 16:39:07 -0300
Subject: [PATCH 375/716] Dupefilter type hints (#5108)

---
 scrapy/dupefilters.py   | 41 +++++++++++++++++++++++++++--------------
 scrapy/utils/request.py |  2 +-
 2 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index ac5478e7c..292c68099 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,35 +1,47 @@
-import os
 import logging
+import os
+from typing import Optional, Set, Type, TypeVar
 
+from twisted.internet.defer import Deferred
+
+from scrapy.http.request import Request
+from scrapy.settings import BaseSettings
+from scrapy.spiders import Spider
 from scrapy.utils.job import job_dir
 from scrapy.utils.request import referer_str, request_fingerprint
 
 
-class BaseDupeFilter:
+BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
 
+
+class BaseDupeFilter:
     @classmethod
-    def from_settings(cls, settings):
+    def from_settings(cls: Type[BaseDupeFilterTV], settings: BaseSettings) -> BaseDupeFilterTV:
         return cls()
 
-    def request_seen(self, request):
+    def request_seen(self, request: Request) -> bool:
         return False
 
-    def open(self):  # can return deferred
+    def open(self) -> Optional[Deferred]:
         pass
 
-    def close(self, reason):  # can return a deferred
+    def close(self, reason: str) -> Optional[Deferred]:
         pass
 
-    def log(self, request, spider):  # log that a request has been filtered
+    def log(self, request: Request, spider: Spider) -> None:
+        """Log that a request has been filtered"""
         pass
 
 
+RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
+
+
 class RFPDupeFilter(BaseDupeFilter):
     """Request Fingerprint duplicates filter"""
 
-    def __init__(self, path=None, debug=False):
+    def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:
         self.file = None
-        self.fingerprints = set()
+        self.fingerprints: Set[str] = set()
         self.logdupes = True
         self.debug = debug
         self.logger = logging.getLogger(__name__)
@@ -39,26 +51,27 @@ class RFPDupeFilter(BaseDupeFilter):
             self.fingerprints.update(x.rstrip() for x in self.file)
 
     @classmethod
-    def from_settings(cls, settings):
+    def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:
         debug = settings.getbool('DUPEFILTER_DEBUG')
         return cls(job_dir(settings), debug)
 
-    def request_seen(self, request):
+    def request_seen(self, request: Request) -> bool:
         fp = self.request_fingerprint(request)
         if fp in self.fingerprints:
             return True
         self.fingerprints.add(fp)
         if self.file:
             self.file.write(fp + '\n')
+        return False
 
-    def request_fingerprint(self, request):
+    def request_fingerprint(self, request: Request) -> str:
         return request_fingerprint(request)
 
-    def close(self, reason):
+    def close(self, reason: str) -> None:
         if self.file:
             self.file.close()
 
-    def log(self, request, spider):
+    def log(self, request: Request, spider: Spider) -> None:
         if self.debug:
             msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
             args = {'request': request, 'referer': referer_str(request)}
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 66736b42f..541368423 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -24,7 +24,7 @@ def request_fingerprint(
     request: Request,
     include_headers: Optional[Iterable[Union[bytes, str]]] = None,
     keep_fragments: bool = False,
-):
+) -> str:
     """
     Return the request fingerprint.
 

From 68379197986ae3deb81a545b5fd6920ea3347094 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Mon, 26 Apr 2021 14:55:02 -0300
Subject: [PATCH 376/716] Add peek method to queues (#5112)

---
 pylintrc                      |   1 +
 scrapy/pqueues.py             |  59 +++++++---
 scrapy/squeues.py             |  59 +++++++---
 tests/test_pqueues.py         | 144 +++++++++++++++++++++++
 tests/test_squeues_request.py | 214 ++++++++++++++++++++++++++++++++++
 5 files changed, 447 insertions(+), 30 deletions(-)
 create mode 100644 tests/test_pqueues.py
 create mode 100644 tests/test_squeues_request.py

diff --git a/pylintrc b/pylintrc
index 5b6b9fab0..972bf99de 100644
--- a/pylintrc
+++ b/pylintrc
@@ -24,6 +24,7 @@ disable=abstract-method,
         consider-using-in,
         consider-using-set-comprehension,
         consider-using-sys-exit,
+        consider-using-with,
         cyclic-import,
         dangerous-default-value,
         deprecated-method,
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index a9aa6c649..b4b63e7c7 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -3,6 +3,7 @@ import logging
 
 from scrapy.utils.misc import create_instance
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -17,8 +18,7 @@ def _path_safe(text):
     >>> _path_safe('some@symbol?').startswith('some_symbol_')
     True
     """
-    pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_'
-                             for c in text])
+    pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in text])
     # as we replace some letters we can get collision for different slots
     # add we add unique part
     unique_slot = hashlib.md5(text.encode('utf8')).hexdigest()
@@ -35,6 +35,9 @@ class ScrapyPriorityQueue:
         * close()
         * __len__()
 
+    Optionally, the queue could provide a ``peek`` method, that should return the
+    next object to be returned by ``pop``, but without removing it from the queue.
+
     ``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls
     argument, which is a class used to instantiate a new (internal) queue when
     a new priority is allocated.
@@ -70,10 +73,12 @@ class ScrapyPriorityQueue:
         self.curprio = min(startprios)
 
     def qfactory(self, key):
-        return create_instance(self.downstream_queue_cls,
-                               None,
-                               self.crawler,
-                               self.key + '/' + str(key))
+        return create_instance(
+            self.downstream_queue_cls,
+            None,
+            self.crawler,
+            self.key + '/' + str(key),
+        )
 
     def priority(self, request):
         return -request.priority
@@ -99,6 +104,18 @@ class ScrapyPriorityQueue:
             self.curprio = min(prios) if prios else None
         return m
 
+    def peek(self):
+        """Returns the next object to be returned by :meth:`pop`,
+        but without removing it from the queue.
+
+        Raises :exc:`NotImplementedError` if the underlying queue class does
+        not implement a ``peek`` method, which is optional for queues.
+        """
+        if self.curprio is None:
+            return None
+        queue = self.queues[self.curprio]
+        return queue.peek()
+
     def close(self):
         active = []
         for p, q in self.queues.items():
@@ -116,8 +133,7 @@ class DownloaderInterface:
         self.downloader = crawler.engine.downloader
 
     def stats(self, possible_slots):
-        return [(self._active_downloads(slot), slot)
-                for slot in possible_slots]
+        return [(self._active_downloads(slot), slot) for slot in possible_slots]
 
     def get_slot_key(self, request):
         return self.downloader._get_slot_key(request, None)
@@ -162,10 +178,12 @@ class DownloaderAwarePriorityQueue:
             self.pqueues[slot] = self.pqfactory(slot, startprios)
 
     def pqfactory(self, slot, startprios=()):
-        return ScrapyPriorityQueue(self.crawler,
-                                   self.downstream_queue_cls,
-                                   self.key + '/' + _path_safe(slot),
-                                   startprios)
+        return ScrapyPriorityQueue(
+            self.crawler,
+            self.downstream_queue_cls,
+            self.key + '/' + _path_safe(slot),
+            startprios,
+        )
 
     def pop(self):
         stats = self._downloader_interface.stats(self.pqueues)
@@ -187,9 +205,22 @@ class DownloaderAwarePriorityQueue:
         queue = self.pqueues[slot]
         queue.push(request)
 
+    def peek(self):
+        """Returns the next object to be returned by :meth:`pop`,
+        but without removing it from the queue.
+
+        Raises :exc:`NotImplementedError` if the underlying queue class does
+        not implement a ``peek`` method, which is optional for queues.
+        """
+        stats = self._downloader_interface.stats(self.pqueues)
+        if not stats:
+            return None
+        slot = min(stats)[1]
+        queue = self.pqueues[slot]
+        return queue.peek()
+
     def close(self):
-        active = {slot: queue.close()
-                  for slot, queue in self.pqueues.items()}
+        active = {slot: queue.close() for slot, queue in self.pqueues.items()}
         self.pqueues.clear()
         return active
 
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 77ffda6f7..44898ba08 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -19,7 +19,6 @@ def _with_mkdir(queue_class):
             dirname = os.path.dirname(path)
             if not os.path.exists(dirname):
                 os.makedirs(dirname, exist_ok=True)
-
             super().__init__(path, *args, **kwargs)
 
     return DirectoriesCreated
@@ -38,6 +37,20 @@ def _serializable_queue(queue_class, serialize, deserialize):
             if s:
                 return deserialize(s)
 
+        def peek(self):
+            """Returns the next object to be returned by :meth:`pop`,
+            but without removing it from the queue.
+
+            Raises :exc:`NotImplementedError` if the underlying queue class does
+            not implement a ``peek`` method, which is optional for queues.
+            """
+            try:
+                s = super().peek()
+            except AttributeError as ex:
+                raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex
+            if s:
+                return deserialize(s)
+
     return SerializableQueue
 
 
@@ -59,12 +72,21 @@ def _scrapy_serialization_queue(queue_class):
 
         def pop(self):
             request = super().pop()
-
             if not request:
                 return None
+            return request_from_dict(request, self.spider)
 
-            request = request_from_dict(request, self.spider)
-            return request
+        def peek(self):
+            """Returns the next object to be returned by :meth:`pop`,
+            but without removing it from the queue.
+
+            Raises :exc:`NotImplementedError` if the underlying queue class does
+            not implement a ``peek`` method, which is optional for queues.
+            """
+            request = super().peek()
+            if not request:
+                return None
+            return request_from_dict(request, self.spider)
 
     return ScrapyRequestQueue
 
@@ -76,6 +98,19 @@ def _scrapy_non_serialization_queue(queue_class):
         def from_crawler(cls, crawler, *args, **kwargs):
             return cls()
 
+        def peek(self):
+            """Returns the next object to be returned by :meth:`pop`,
+            but without removing it from the queue.
+
+            Raises :exc:`NotImplementedError` if the underlying queue class does
+            not implement a ``peek`` method, which is optional for queues.
+            """
+            try:
+                s = super().peek()
+            except AttributeError as ex:
+                raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex
+            return s
+
     return ScrapyRequestQueue
 
 
@@ -109,17 +144,9 @@ MarshalLifoDiskQueueNonRequest = _serializable_queue(
     marshal.loads
 )
 
-PickleFifoDiskQueue = _scrapy_serialization_queue(
-    PickleFifoDiskQueueNonRequest
-)
-PickleLifoDiskQueue = _scrapy_serialization_queue(
-    PickleLifoDiskQueueNonRequest
-)
-MarshalFifoDiskQueue = _scrapy_serialization_queue(
-    MarshalFifoDiskQueueNonRequest
-)
-MarshalLifoDiskQueue = _scrapy_serialization_queue(
-    MarshalLifoDiskQueueNonRequest
-)
+PickleFifoDiskQueue = _scrapy_serialization_queue(PickleFifoDiskQueueNonRequest)
+PickleLifoDiskQueue = _scrapy_serialization_queue(PickleLifoDiskQueueNonRequest)
+MarshalFifoDiskQueue = _scrapy_serialization_queue(MarshalFifoDiskQueueNonRequest)
+MarshalLifoDiskQueue = _scrapy_serialization_queue(MarshalLifoDiskQueueNonRequest)
 FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)
 LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)
diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py
new file mode 100644
index 000000000..ec55033d1
--- /dev/null
+++ b/tests/test_pqueues.py
@@ -0,0 +1,144 @@
+import tempfile
+import unittest
+
+import queuelib
+
+from scrapy.http.request import Request
+from scrapy.pqueues import ScrapyPriorityQueue, DownloaderAwarePriorityQueue
+from scrapy.spiders import Spider
+from scrapy.squeues import FifoMemoryQueue
+from scrapy.utils.test import get_crawler
+
+from tests.test_scheduler import MockDownloader, MockEngine
+
+
+class PriorityQueueTest(unittest.TestCase):
+    def setUp(self):
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider("foo")
+
+    def test_queue_push_pop_one(self):
+        temp_dir = tempfile.mkdtemp()
+        queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir)
+        self.assertIsNone(queue.pop())
+        self.assertEqual(len(queue), 0)
+        req1 = Request("https://example.org/1", priority=1)
+        queue.push(req1)
+        self.assertEqual(len(queue), 1)
+        dequeued = queue.pop()
+        self.assertEqual(len(queue), 0)
+        self.assertEqual(dequeued.url, req1.url)
+        self.assertEqual(dequeued.priority, req1.priority)
+        self.assertEqual(queue.close(), [])
+
+    def test_no_peek_raises(self):
+        if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+            raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined")
+        temp_dir = tempfile.mkdtemp()
+        queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir)
+        queue.push(Request("https://example.org"))
+        with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+            queue.peek()
+        queue.close()
+
+    def test_peek(self):
+        if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+            raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined")
+        temp_dir = tempfile.mkdtemp()
+        queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir)
+        self.assertEqual(len(queue), 0)
+        self.assertIsNone(queue.peek())
+        req1 = Request("https://example.org/1")
+        req2 = Request("https://example.org/2")
+        req3 = Request("https://example.org/3")
+        queue.push(req1)
+        queue.push(req2)
+        queue.push(req3)
+        self.assertEqual(len(queue), 3)
+        self.assertEqual(queue.peek().url, req1.url)
+        self.assertEqual(queue.pop().url, req1.url)
+        self.assertEqual(len(queue), 2)
+        self.assertEqual(queue.peek().url, req2.url)
+        self.assertEqual(queue.pop().url, req2.url)
+        self.assertEqual(len(queue), 1)
+        self.assertEqual(queue.peek().url, req3.url)
+        self.assertEqual(queue.pop().url, req3.url)
+        self.assertEqual(queue.close(), [])
+
+    def test_queue_push_pop_priorities(self):
+        temp_dir = tempfile.mkdtemp()
+        queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3])
+        self.assertIsNone(queue.pop())
+        self.assertEqual(len(queue), 0)
+        req1 = Request("https://example.org/1", priority=1)
+        req2 = Request("https://example.org/2", priority=2)
+        req3 = Request("https://example.org/3", priority=3)
+        queue.push(req1)
+        queue.push(req2)
+        queue.push(req3)
+        self.assertEqual(len(queue), 3)
+        dequeued = queue.pop()
+        self.assertEqual(len(queue), 2)
+        self.assertEqual(dequeued.url, req3.url)
+        self.assertEqual(dequeued.priority, req3.priority)
+        self.assertEqual(queue.close(), [-1, -2])
+
+
+class DownloaderAwarePriorityQueueTest(unittest.TestCase):
+    def setUp(self):
+        crawler = get_crawler(Spider)
+        crawler.engine = MockEngine(downloader=MockDownloader())
+        self.queue = DownloaderAwarePriorityQueue.from_crawler(
+            crawler=crawler,
+            downstream_queue_cls=FifoMemoryQueue,
+            key="foo/bar",
+        )
+
+    def tearDown(self):
+        self.queue.close()
+
+    def test_push_pop(self):
+        self.assertEqual(len(self.queue), 0)
+        self.assertIsNone(self.queue.pop())
+        req1 = Request("http://www.example.com/1")
+        req2 = Request("http://www.example.com/2")
+        req3 = Request("http://www.example.com/3")
+        self.queue.push(req1)
+        self.queue.push(req2)
+        self.queue.push(req3)
+        self.assertEqual(len(self.queue), 3)
+        self.assertEqual(self.queue.pop().url, req1.url)
+        self.assertEqual(len(self.queue), 2)
+        self.assertEqual(self.queue.pop().url, req2.url)
+        self.assertEqual(len(self.queue), 1)
+        self.assertEqual(self.queue.pop().url, req3.url)
+        self.assertEqual(len(self.queue), 0)
+        self.assertIsNone(self.queue.pop())
+
+    def test_no_peek_raises(self):
+        if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+            raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined")
+        self.queue.push(Request("https://example.org"))
+        with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+            self.queue.peek()
+
+    def test_peek(self):
+        if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+            raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined")
+        self.assertEqual(len(self.queue), 0)
+        req1 = Request("https://example.org/1")
+        req2 = Request("https://example.org/2")
+        req3 = Request("https://example.org/3")
+        self.queue.push(req1)
+        self.queue.push(req2)
+        self.queue.push(req3)
+        self.assertEqual(len(self.queue), 3)
+        self.assertEqual(self.queue.peek().url, req1.url)
+        self.assertEqual(self.queue.pop().url, req1.url)
+        self.assertEqual(len(self.queue), 2)
+        self.assertEqual(self.queue.peek().url, req2.url)
+        self.assertEqual(self.queue.pop().url, req2.url)
+        self.assertEqual(len(self.queue), 1)
+        self.assertEqual(self.queue.peek().url, req3.url)
+        self.assertEqual(self.queue.pop().url, req3.url)
+        self.assertIsNone(self.queue.peek())
diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py
new file mode 100644
index 000000000..c5fcc1853
--- /dev/null
+++ b/tests/test_squeues_request.py
@@ -0,0 +1,214 @@
+import shutil
+import tempfile
+import unittest
+
+import queuelib
+
+from scrapy.squeues import (
+    PickleFifoDiskQueue,
+    PickleLifoDiskQueue,
+    MarshalFifoDiskQueue,
+    MarshalLifoDiskQueue,
+    FifoMemoryQueue,
+    LifoMemoryQueue,
+)
+from scrapy.http import Request
+from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+
+
+"""
+Queues that handle requests
+"""
+
+
+class BaseQueueTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-")
+        self.qpath = self.tempfilename()
+        self.qdir = self.mkdtemp()
+        self.crawler = get_crawler(Spider)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+
+    def tempfilename(self):
+        with tempfile.NamedTemporaryFile(dir=self.tmpdir) as nf:
+            return nf.name
+
+    def mkdtemp(self):
+        return tempfile.mkdtemp(dir=self.tmpdir)
+
+
+class RequestQueueTestMixin:
+    def queue(self):
+        raise NotImplementedError()
+
+    def test_one_element_with_peek(self):
+        if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+            raise unittest.SkipTest("The queuelib queues do not define peek")
+        q = self.queue()
+        self.assertEqual(len(q), 0)
+        self.assertIsNone(q.peek())
+        self.assertIsNone(q.pop())
+        req = Request("http://www.example.com")
+        q.push(req)
+        self.assertEqual(len(q), 1)
+        self.assertEqual(q.peek().url, req.url)
+        self.assertEqual(q.pop().url, req.url)
+        self.assertEqual(len(q), 0)
+        self.assertIsNone(q.peek())
+        self.assertIsNone(q.pop())
+        q.close()
+
+    def test_one_element_without_peek(self):
+        if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+            raise unittest.SkipTest("The queuelib queues define peek")
+        q = self.queue()
+        self.assertEqual(len(q), 0)
+        self.assertIsNone(q.pop())
+        req = Request("http://www.example.com")
+        q.push(req)
+        self.assertEqual(len(q), 1)
+        with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+            q.peek()
+        self.assertEqual(q.pop().url, req.url)
+        self.assertEqual(len(q), 0)
+        self.assertIsNone(q.pop())
+        q.close()
+
+
+class FifoQueueMixin(RequestQueueTestMixin):
+    def test_fifo_with_peek(self):
+        if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+            raise unittest.SkipTest("The queuelib queues do not define peek")
+        q = self.queue()
+        self.assertEqual(len(q), 0)
+        self.assertIsNone(q.peek())
+        self.assertIsNone(q.pop())
+        req1 = Request("http://www.example.com/1")
+        req2 = Request("http://www.example.com/2")
+        req3 = Request("http://www.example.com/3")
+        q.push(req1)
+        q.push(req2)
+        q.push(req3)
+        self.assertEqual(len(q), 3)
+        self.assertEqual(q.peek().url, req1.url)
+        self.assertEqual(q.pop().url, req1.url)
+        self.assertEqual(len(q), 2)
+        self.assertEqual(q.peek().url, req2.url)
+        self.assertEqual(q.pop().url, req2.url)
+        self.assertEqual(len(q), 1)
+        self.assertEqual(q.peek().url, req3.url)
+        self.assertEqual(q.pop().url, req3.url)
+        self.assertEqual(len(q), 0)
+        self.assertIsNone(q.peek())
+        self.assertIsNone(q.pop())
+        q.close()
+
+    def test_fifo_without_peek(self):
+        if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+            raise unittest.SkipTest("The queuelib queues do not define peek")
+        q = self.queue()
+        self.assertEqual(len(q), 0)
+        self.assertIsNone(q.pop())
+        req1 = Request("http://www.example.com/1")
+        req2 = Request("http://www.example.com/2")
+        req3 = Request("http://www.example.com/3")
+        q.push(req1)
+        q.push(req2)
+        q.push(req3)
+        with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+            q.peek()
+        self.assertEqual(len(q), 3)
+        self.assertEqual(q.pop().url, req1.url)
+        self.assertEqual(len(q), 2)
+        self.assertEqual(q.pop().url, req2.url)
+        self.assertEqual(len(q), 1)
+        self.assertEqual(q.pop().url, req3.url)
+        self.assertEqual(len(q), 0)
+        self.assertIsNone(q.pop())
+        q.close()
+
+
+class LifoQueueMixin(RequestQueueTestMixin):
+    def test_lifo_with_peek(self):
+        if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+            raise unittest.SkipTest("The queuelib queues do not define peek")
+        q = self.queue()
+        self.assertEqual(len(q), 0)
+        self.assertIsNone(q.peek())
+        self.assertIsNone(q.pop())
+        req1 = Request("http://www.example.com/1")
+        req2 = Request("http://www.example.com/2")
+        req3 = Request("http://www.example.com/3")
+        q.push(req1)
+        q.push(req2)
+        q.push(req3)
+        self.assertEqual(len(q), 3)
+        self.assertEqual(q.peek().url, req3.url)
+        self.assertEqual(q.pop().url, req3.url)
+        self.assertEqual(len(q), 2)
+        self.assertEqual(q.peek().url, req2.url)
+        self.assertEqual(q.pop().url, req2.url)
+        self.assertEqual(len(q), 1)
+        self.assertEqual(q.peek().url, req1.url)
+        self.assertEqual(q.pop().url, req1.url)
+        self.assertEqual(len(q), 0)
+        self.assertIsNone(q.peek())
+        self.assertIsNone(q.pop())
+        q.close()
+
+    def test_lifo_without_peek(self):
+        if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
+            raise unittest.SkipTest("The queuelib queues do not define peek")
+        q = self.queue()
+        self.assertEqual(len(q), 0)
+        self.assertIsNone(q.pop())
+        req1 = Request("http://www.example.com/1")
+        req2 = Request("http://www.example.com/2")
+        req3 = Request("http://www.example.com/3")
+        q.push(req1)
+        q.push(req2)
+        q.push(req3)
+        with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+            q.peek()
+        self.assertEqual(len(q), 3)
+        self.assertEqual(q.pop().url, req3.url)
+        self.assertEqual(len(q), 2)
+        self.assertEqual(q.pop().url, req2.url)
+        self.assertEqual(len(q), 1)
+        self.assertEqual(q.pop().url, req1.url)
+        self.assertEqual(len(q), 0)
+        self.assertIsNone(q.pop())
+        q.close()
+
+
+class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
+    def queue(self):
+        return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo")
+
+
+class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
+    def queue(self):
+        return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo")
+
+
+class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
+    def queue(self):
+        return MarshalFifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/fifo")
+
+
+class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
+    def queue(self):
+        return MarshalLifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/lifo")
+
+
+class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
+    def queue(self):
+        return FifoMemoryQueue.from_crawler(crawler=self.crawler)
+
+
+class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
+    def queue(self):
+        return LifoMemoryQueue.from_crawler(crawler=self.crawler)

From ddea6b7bfa38bf5402d78350ab61e2c827ca49b5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Mon, 26 Apr 2021 16:16:14 -0300
Subject: [PATCH 377/716] Scheduler: minimal interface, API docs (#3559)

---
 docs/index.rst               |   4 +
 docs/topics/architecture.rst |   5 +-
 docs/topics/scheduler.rst    |  34 ++++
 docs/topics/settings.rst     |   3 +-
 scrapy/core/engine.py        |  21 ++-
 scrapy/core/scheduler.py     | 298 +++++++++++++++++++++++++++--------
 scrapy/utils/job.py          |   5 +-
 tests/test_scheduler_base.py | 159 +++++++++++++++++++
 8 files changed, 458 insertions(+), 71 deletions(-)
 create mode 100644 docs/topics/scheduler.rst
 create mode 100644 tests/test_scheduler_base.py

diff --git a/docs/index.rst b/docs/index.rst
index da264fb34..433798aa8 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -227,6 +227,7 @@ Extending Scrapy
    topics/extensions
    topics/api
    topics/signals
+   topics/scheduler
    topics/exporters
 
 
@@ -248,6 +249,9 @@ Extending Scrapy
 :doc:`topics/signals`
     See all available signals and how to work with them.
 
+:doc:`topics/scheduler`
+    Understand the scheduler component.
+
 :doc:`topics/exporters`
     Quickly export your scraped items to a file (XML, CSV, etc).
 
diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 074c59241..71d027c86 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -87,8 +87,9 @@ of the system, and triggering events when certain actions occur. See the
 Scheduler
 ---------
 
-The Scheduler receives requests from the engine and enqueues them for feeding
-them later (also to the engine) when the engine requests them.
+The :ref:`scheduler <topics-scheduler>` receives requests from the engine and 
+enqueues them for feeding them later (also to the engine) when the engine 
+requests them.
 
 .. _component-downloader:
 
diff --git a/docs/topics/scheduler.rst b/docs/topics/scheduler.rst
new file mode 100644
index 000000000..57c24b76a
--- /dev/null
+++ b/docs/topics/scheduler.rst
@@ -0,0 +1,34 @@
+.. _topics-scheduler:
+
+=========
+Scheduler
+=========
+
+.. module:: scrapy.core.scheduler
+
+The scheduler component receives requests from the :ref:`engine <component-engine>`
+and stores them into persistent and/or non-persistent data structures.
+It also gets those requests and feeds them back to the engine when it
+asks for a next request to be downloaded.
+
+
+Overriding the default scheduler
+================================
+
+You can use your own custom scheduler class by supplying its full
+Python path in the :setting:`SCHEDULER` setting.
+
+
+Minimal scheduler interface
+===========================
+
+.. autoclass:: BaseScheduler
+   :members:
+
+
+Default Scrapy scheduler
+========================
+
+.. autoclass:: Scheduler
+   :members:
+   :special-members: __len__
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 1d5babcec..e4fb2baf7 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1280,7 +1280,8 @@ SCHEDULER
 
 Default: ``'scrapy.core.scheduler.Scheduler'``
 
-The scheduler to use for crawling.
+The scheduler class to be used for crawling.
+See the :ref:`topics-scheduler` topic for details.
 
 .. setting:: SCHEDULER_DEBUG
 
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 7a09bafa1..dd3225082 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -17,6 +17,7 @@ from scrapy import signals
 from scrapy.core.scraper import Scraper
 from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning
 from scrapy.http import Response, Request
+from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider
 from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
 from scrapy.utils.misc import create_instance, load_object
@@ -73,12 +74,22 @@ class ExecutionEngine:
         self.spider: Optional[Spider] = None
         self.running = False
         self.paused = False
-        self.scheduler_cls = load_object(crawler.settings["SCHEDULER"])
+        self.scheduler_cls = self._get_scheduler_class(crawler.settings)
         downloader_cls = load_object(self.settings['DOWNLOADER'])
         self.downloader = downloader_cls(crawler)
         self.scraper = Scraper(crawler)
         self._spider_closed_callback = spider_closed_callback
 
+    def _get_scheduler_class(self, settings: BaseSettings) -> type:
+        from scrapy.core.scheduler import BaseScheduler
+        scheduler_cls = load_object(settings["SCHEDULER"])
+        if not issubclass(scheduler_cls, BaseScheduler):
+            raise TypeError(
+                f"The provided scheduler class ({settings['SCHEDULER']})"
+                " does not fully implement the scheduler interface"
+            )
+        return scheduler_cls
+
     @inlineCallbacks
     def start(self) -> Deferred:
         if self.running:
@@ -301,7 +312,8 @@ class ExecutionEngine:
         start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider)
         self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
         self.spider = spider
-        yield scheduler.open(spider)
+        if hasattr(scheduler, "open"):
+            yield scheduler.open(spider)
         yield self.scraper.open_spider(spider)
         self.crawler.stats.open_spider(spider)
         yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
@@ -345,8 +357,9 @@ class ExecutionEngine:
         dfd.addBoth(lambda _: self.scraper.close_spider(spider))
         dfd.addErrback(log_failure('Scraper close failure'))
 
-        dfd.addBoth(lambda _: self.slot.scheduler.close(reason))
-        dfd.addErrback(log_failure('Scheduler close failure'))
+        if hasattr(self.slot.scheduler, "close"):
+            dfd.addBoth(lambda _: self.slot.scheduler.close(reason))
+            dfd.addErrback(log_failure("Scheduler close failure"))
 
         dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(
             signal=signals.spider_closed, spider=spider, reason=reason,
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 9ce823dbc..5ba0fb63b 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -1,42 +1,179 @@
-import os
 import json
 import logging
-from os.path import join, exists
+import os
+from abc import abstractmethod
+from os.path import exists, join
+from typing import Optional, Type, TypeVar
 
-from scrapy.utils.misc import load_object, create_instance
+from twisted.internet.defer import Deferred
+
+from scrapy.crawler import Crawler
+from scrapy.http.request import Request
+from scrapy.spiders import Spider
 from scrapy.utils.job import job_dir
+from scrapy.utils.misc import create_instance, load_object
 
 
 logger = logging.getLogger(__name__)
 
 
-class Scheduler:
+class BaseSchedulerMeta(type):
     """
-    Scrapy Scheduler. It allows to enqueue requests and then get
-    a next request to download. Scheduler is also handling duplication
-    filtering, via dupefilter.
-
-    Prioritization and queueing is not performed by the Scheduler.
-    User sets ``priority`` field for each Request, and a PriorityQueue
-    (defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities
-    to dequeue requests in a desired order.
-
-    Scheduler uses two PriorityQueue instances, configured to work in-memory
-    and on-disk (optional). When on-disk queue is present, it is used by
-    default, and an in-memory queue is used as a fallback for cases where
-    a disk queue can't handle a request (can't serialize it).
-
-    :setting:`SCHEDULER_MEMORY_QUEUE` and
-    :setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes
-    which PriorityQueue instances would be instantiated with, to keep requests
-    on disk and in memory respectively.
-
-    Overall, Scheduler is an object which holds several PriorityQueue instances
-    (in-memory and on-disk) and implements fallback logic for them.
-    Also, it handles dupefilters.
+    Metaclass to check scheduler classes against the necessary interface
     """
-    def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None,
-                 logunser=False, stats=None, pqclass=None, crawler=None):
+    def __instancecheck__(cls, instance):
+        return cls.__subclasscheck__(type(instance))
+
+    def __subclasscheck__(cls, subclass):
+        return (
+            hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests)
+            and hasattr(subclass, "enqueue_request") and callable(subclass.enqueue_request)
+            and hasattr(subclass, "next_request") and callable(subclass.next_request)
+        )
+
+
+class BaseScheduler(metaclass=BaseSchedulerMeta):
+    """
+    The scheduler component is responsible for storing requests received from
+    the engine, and feeding them back upon request (also to the engine).
+
+    The original sources of said requests are:
+
+    * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks
+    * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods
+    * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods
+
+    The order in which the scheduler returns its stored requests (via the ``next_request`` method)
+    plays a great part in determining the order in which those requests are downloaded.
+
+    The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with.
+    """
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler):
+        """
+        Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument.
+        """
+        return cls()
+
+    def open(self, spider: Spider) -> Optional[Deferred]:
+        """
+        Called when the spider is opened by the engine. It receives the spider
+        instance as argument and it's useful to execute initialization code.
+
+        :param spider: the spider object for the current crawl
+        :type spider: :class:`~scrapy.spiders.Spider`
+        """
+        pass
+
+    def close(self, reason: str) -> Optional[Deferred]:
+        """
+        Called when the spider is closed by the engine. It receives the reason why the crawl
+        finished as argument and it's useful to execute cleaning code.
+
+        :param reason: a string which describes the reason why the spider was closed
+        :type reason: :class:`str`
+        """
+        pass
+
+    @abstractmethod
+    def has_pending_requests(self) -> bool:
+        """
+        ``True`` if the scheduler has enqueued requests, ``False`` otherwise
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def enqueue_request(self, request: Request) -> bool:
+        """
+        Process a request received by the engine.
+
+        Return ``True`` if the request is stored correctly, ``False`` otherwise.
+
+        If ``False``, the engine will fire a ``request_dropped`` signal, and
+        will not make further attempts to schedule the request at a later time.
+        For reference, the default Scrapy scheduler returns ``False`` when the
+        request is rejected by the dupefilter.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def next_request(self) -> Optional[Request]:
+        """
+        Return the next :class:`~scrapy.http.Request` to be processed, or ``None``
+        to indicate that there are no requests to be considered ready at the moment.
+
+        Returning ``None`` implies that no request from the scheduler will be sent
+        to the downloader in the current reactor cycle. The engine will continue
+        calling ``next_request`` until ``has_pending_requests`` is ``False``.
+        """
+        raise NotImplementedError()
+
+
+SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler")
+
+
+class Scheduler(BaseScheduler):
+    """
+    Default Scrapy scheduler. This implementation also handles duplication
+    filtering via the :setting:`dupefilter <DUPEFILTER_CLASS>`.
+
+    This scheduler stores requests into several priority queues (defined by the
+    :setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues
+    are backed by either memory or disk based queues (respectively defined by the
+    :setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings).
+
+    Request prioritization is almost entirely delegated to the priority queue. The only
+    prioritization performed by this scheduler is using the disk-based queue if present
+    (i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based
+    queue if a serialization error occurs. If the disk queue is not present, the memory one
+    is used directly.
+
+    :param dupefilter: An object responsible for checking and filtering duplicate requests.
+                       The value for the :setting:`DUPEFILTER_CLASS` setting is used by default.
+    :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar:
+                      any class that implements the `BaseDupeFilter` interface
+
+    :param jobdir: The path of a directory to be used for persisting the crawl's state.
+                   The value for the :setting:`JOBDIR` setting is used by default.
+                   See :ref:`topics-jobs`.
+    :type jobdir: :class:`str` or ``None``
+
+    :param dqclass: A class to be used as persistent request queue.
+                    The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default.
+    :type dqclass: class
+
+    :param mqclass: A class to be used as non-persistent request queue.
+                    The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default.
+    :type mqclass: class
+
+    :param logunser: A boolean that indicates whether or not unserializable requests should be logged.
+                     The value for the :setting:`SCHEDULER_DEBUG` setting is used by default.
+    :type logunser: bool
+
+    :param stats: A stats collector object to record stats about the request scheduling process.
+                  The value for the :setting:`STATS_CLASS` setting is used by default.
+    :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar:
+                 any class that implements the `StatsCollector` interface
+
+    :param pqclass: A class to be used as priority queue for requests.
+                    The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default.
+    :type pqclass: class
+
+    :param crawler: The crawler object corresponding to the current crawl.
+    :type crawler: :class:`scrapy.crawler.Crawler`
+    """
+    def __init__(
+        self,
+        dupefilter,
+        jobdir: Optional[str] = None,
+        dqclass=None,
+        mqclass=None,
+        logunser: bool = False,
+        stats=None,
+        pqclass=None,
+        crawler: Optional[Crawler] = None,
+    ):
         self.df = dupefilter
         self.dqdir = self._dqdir(jobdir)
         self.pqclass = pqclass
@@ -47,34 +184,57 @@ class Scheduler:
         self.crawler = crawler
 
     @classmethod
-    def from_crawler(cls, crawler):
-        settings = crawler.settings
-        dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
-        dupefilter = create_instance(dupefilter_cls, settings, crawler)
-        pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
-        dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
-        mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
-        logunser = settings.getbool('SCHEDULER_DEBUG')
-        return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
-                   stats=crawler.stats, pqclass=pqclass, dqclass=dqclass,
-                   mqclass=mqclass, crawler=crawler)
+    def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV:
+        """
+        Factory method, initializes the scheduler with arguments taken from the crawl settings
+        """
+        dupefilter_cls = load_object(crawler.settings['DUPEFILTER_CLASS'])
+        return cls(
+            dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler),
+            jobdir=job_dir(crawler.settings),
+            dqclass=load_object(crawler.settings['SCHEDULER_DISK_QUEUE']),
+            mqclass=load_object(crawler.settings['SCHEDULER_MEMORY_QUEUE']),
+            logunser=crawler.settings.getbool('SCHEDULER_DEBUG'),
+            stats=crawler.stats,
+            pqclass=load_object(crawler.settings['SCHEDULER_PRIORITY_QUEUE']),
+            crawler=crawler,
+        )
 
-    def has_pending_requests(self):
+    def has_pending_requests(self) -> bool:
         return len(self) > 0
 
-    def open(self, spider):
+    def open(self, spider: Spider) -> Optional[Deferred]:
+        """
+        (1) initialize the memory queue
+        (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
+        (3) return the result of the dupefilter's ``open`` method
+        """
         self.spider = spider
         self.mqs = self._mq()
         self.dqs = self._dq() if self.dqdir else None
         return self.df.open()
 
-    def close(self, reason):
-        if self.dqs:
+    def close(self, reason: str) -> Optional[Deferred]:
+        """
+        (1) dump pending requests to disk if there is a disk queue
+        (2) return the result of the dupefilter's ``close`` method
+        """
+        if self.dqs is not None:
             state = self.dqs.close()
+            assert isinstance(self.dqdir, str)
             self._write_dqs_state(self.dqdir, state)
         return self.df.close(reason)
 
-    def enqueue_request(self, request):
+    def enqueue_request(self, request: Request) -> bool:
+        """
+        Unless the received request is filtered out by the Dupefilter, attempt to push
+        it into the disk queue, falling back to pushing it into the memory queue.
+
+        Increment the appropriate stats, such as: ``scheduler/enqueued``,
+        ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``.
+
+        Return ``True`` if the request was stored successfully, ``False`` otherwise.
+        """
         if not request.dont_filter and self.df.request_seen(request):
             self.df.log(request, self.spider)
             return False
@@ -87,24 +247,35 @@ class Scheduler:
         self.stats.inc_value('scheduler/enqueued', spider=self.spider)
         return True
 
-    def next_request(self):
+    def next_request(self) -> Optional[Request]:
+        """
+        Return a :class:`~scrapy.http.Request` object from the memory queue,
+        falling back to the disk queue if the memory queue is empty.
+        Return ``None`` if there are no more enqueued requests.
+
+        Increment the appropriate stats, such as: ``scheduler/dequeued``,
+        ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``.
+        """
         request = self.mqs.pop()
-        if request:
+        if request is not None:
             self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider)
         else:
             request = self._dqpop()
-            if request:
+            if request is not None:
                 self.stats.inc_value('scheduler/dequeued/disk', spider=self.spider)
-        if request:
+        if request is not None:
             self.stats.inc_value('scheduler/dequeued', spider=self.spider)
         return request
 
-    def __len__(self):
-        return len(self.dqs) + len(self.mqs) if self.dqs else len(self.mqs)
+    def __len__(self) -> int:
+        """
+        Return the total amount of enqueued requests
+        """
+        return len(self.dqs) + len(self.mqs) if self.dqs is not None else len(self.mqs)
 
-    def _dqpush(self, request):
+    def _dqpush(self, request: Request) -> bool:
         if self.dqs is None:
-            return
+            return False
         try:
             self.dqs.push(request)
         except ValueError as e:  # non serializable request
@@ -115,18 +286,18 @@ class Scheduler:
                 logger.warning(msg, {'request': request, 'reason': e},
                                exc_info=True, extra={'spider': self.spider})
                 self.logunser = False
-            self.stats.inc_value('scheduler/unserializable',
-                                 spider=self.spider)
-            return
+            self.stats.inc_value('scheduler/unserializable', spider=self.spider)
+            return False
         else:
             return True
 
-    def _mqpush(self, request):
+    def _mqpush(self, request: Request) -> None:
         self.mqs.push(request)
 
-    def _dqpop(self):
-        if self.dqs:
+    def _dqpop(self) -> Optional[Request]:
+        if self.dqs is not None:
             return self.dqs.pop()
+        return None
 
     def _mq(self):
         """ Create a new priority queue instance, with in-memory storage """
@@ -150,21 +321,22 @@ class Scheduler:
                         {'queuesize': len(q)}, extra={'spider': self.spider})
         return q
 
-    def _dqdir(self, jobdir):
+    def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
         """ Return a folder name to keep disk queue state at """
-        if jobdir:
+        if jobdir is not None:
             dqdir = join(jobdir, 'requests.queue')
             if not exists(dqdir):
                 os.makedirs(dqdir)
             return dqdir
+        return None
 
-    def _read_dqs_state(self, dqdir):
+    def _read_dqs_state(self, dqdir: str) -> list:
         path = join(dqdir, 'active.json')
         if not exists(path):
-            return ()
+            return []
         with open(path) as f:
             return json.load(f)
 
-    def _write_dqs_state(self, dqdir, state):
+    def _write_dqs_state(self, dqdir: str, state: list) -> None:
         with open(join(dqdir, 'active.json'), 'w') as f:
             json.dump(state, f)
diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py
index 4f1e601fc..c92ef36f5 100644
--- a/scrapy/utils/job.py
+++ b/scrapy/utils/job.py
@@ -1,7 +1,10 @@
 import os
+from typing import Optional
+
+from scrapy.settings import BaseSettings
 
 
-def job_dir(settings):
+def job_dir(settings: BaseSettings) -> Optional[str]:
     path = settings['JOBDIR']
     if path and not os.path.exists(path):
         os.makedirs(path)
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
new file mode 100644
index 000000000..bf90b4320
--- /dev/null
+++ b/tests/test_scheduler_base.py
@@ -0,0 +1,159 @@
+from typing import Dict, Optional
+from unittest import TestCase
+from urllib.parse import urljoin, urlparse
+
+from testfixtures import LogCapture
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase as TwistedTestCase
+
+from scrapy.core.scheduler import BaseScheduler
+from scrapy.crawler import CrawlerRunner
+from scrapy.http import Request
+from scrapy.spiders import Spider
+from scrapy.utils.request import request_fingerprint
+
+from tests.mockserver import MockServer
+
+
+PATHS = ["/a", "/b", "/c"]
+URLS = [urljoin("https://example.org", p) for p in PATHS]
+
+
+class MinimalScheduler:
+    def __init__(self) -> None:
+        self.requests: Dict[str, Request] = {}
+
+    def has_pending_requests(self) -> bool:
+        return bool(self.requests)
+
+    def enqueue_request(self, request: Request) -> bool:
+        fp = request_fingerprint(request)
+        if fp not in self.requests:
+            self.requests[fp] = request
+            return True
+        return False
+
+    def next_request(self) -> Optional[Request]:
+        if self.has_pending_requests():
+            fp, request = self.requests.popitem()
+            return request
+        return None
+
+
+class SimpleScheduler(MinimalScheduler):
+    def open(self, spider: Spider) -> defer.Deferred:
+        return defer.succeed("open")
+
+    def close(self, reason: str) -> defer.Deferred:
+        return defer.succeed("close")
+
+    def __len__(self) -> int:
+        return len(self.requests)
+
+
+class TestSpider(Spider):
+    name = "test"
+
+    def __init__(self, mockserver, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.start_urls = map(mockserver.url, PATHS)
+
+    def parse(self, response):
+        return {"path": urlparse(response.url).path}
+
+
+class InterfaceCheckMixin:
+    def test_scheduler_class(self):
+        self.assertTrue(isinstance(self.scheduler, BaseScheduler))
+        self.assertTrue(issubclass(self.scheduler.__class__, BaseScheduler))
+
+
+class BaseSchedulerTest(TestCase, InterfaceCheckMixin):
+    def setUp(self):
+        self.scheduler = BaseScheduler()
+
+    def test_methods(self):
+        self.assertIsNone(self.scheduler.open(Spider("foo")))
+        self.assertIsNone(self.scheduler.close("finished"))
+        self.assertRaises(NotImplementedError, self.scheduler.has_pending_requests)
+        self.assertRaises(NotImplementedError, self.scheduler.enqueue_request, Request("https://example.org"))
+        self.assertRaises(NotImplementedError, self.scheduler.next_request)
+
+
+class MinimalSchedulerTest(TestCase, InterfaceCheckMixin):
+    def setUp(self):
+        self.scheduler = MinimalScheduler()
+
+    def test_open_close(self):
+        with self.assertRaises(AttributeError):
+            self.scheduler.open(Spider("foo"))
+        with self.assertRaises(AttributeError):
+            self.scheduler.close("finished")
+
+    def test_len(self):
+        with self.assertRaises(AttributeError):
+            self.scheduler.__len__()
+        with self.assertRaises(TypeError):
+            len(self.scheduler)
+
+    def test_enqueue_dequeue(self):
+        self.assertFalse(self.scheduler.has_pending_requests())
+        for url in URLS:
+            self.assertTrue(self.scheduler.enqueue_request(Request(url)))
+            self.assertFalse(self.scheduler.enqueue_request(Request(url)))
+        self.assertTrue(self.scheduler.has_pending_requests)
+
+        dequeued = []
+        while self.scheduler.has_pending_requests():
+            request = self.scheduler.next_request()
+            dequeued.append(request.url)
+        self.assertEqual(set(dequeued), set(URLS))
+        self.assertFalse(self.scheduler.has_pending_requests())
+
+
+class SimpleSchedulerTest(TwistedTestCase, InterfaceCheckMixin):
+    def setUp(self):
+        self.scheduler = SimpleScheduler()
+
+    @defer.inlineCallbacks
+    def test_enqueue_dequeue(self):
+        open_result = yield self.scheduler.open(Spider("foo"))
+        self.assertEqual(open_result, "open")
+        self.assertFalse(self.scheduler.has_pending_requests())
+
+        for url in URLS:
+            self.assertTrue(self.scheduler.enqueue_request(Request(url)))
+            self.assertFalse(self.scheduler.enqueue_request(Request(url)))
+
+        self.assertTrue(self.scheduler.has_pending_requests())
+        self.assertEqual(len(self.scheduler), len(URLS))
+
+        dequeued = []
+        while self.scheduler.has_pending_requests():
+            request = self.scheduler.next_request()
+            dequeued.append(request.url)
+        self.assertEqual(set(dequeued), set(URLS))
+
+        self.assertFalse(self.scheduler.has_pending_requests())
+        self.assertEqual(len(self.scheduler), 0)
+
+        close_result = yield self.scheduler.close("")
+        self.assertEqual(close_result, "close")
+
+
+class MinimalSchedulerCrawlTest(TwistedTestCase):
+    scheduler_cls = MinimalScheduler
+
+    @defer.inlineCallbacks
+    def test_crawl(self):
+        with MockServer() as mockserver:
+            settings = {"SCHEDULER": self.scheduler_cls}
+            with LogCapture() as log:
+                yield CrawlerRunner(settings).crawl(TestSpider, mockserver)
+            for path in PATHS:
+                self.assertIn(f"{{'path': '{path}'}}", str(log))
+            self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log))
+
+
+class SimpleSchedulerCrawlTest(MinimalSchedulerCrawlTest):
+    scheduler_cls = SimpleScheduler

From 02ae1deaf499e79dc8ececf4d5dcea6daef0ade0 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Tue, 27 Apr 2021 09:41:44 -0300
Subject: [PATCH 378/716] Deprecate unused squeues (#5117)

---
 scrapy/squeues.py     | 47 +++++++++++++++++++++++++++++++++++--------
 tests/test_squeues.py | 16 +++++++--------
 2 files changed, 47 insertions(+), 16 deletions(-)

diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 44898ba08..16f7bf4b6 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -8,6 +8,7 @@ import pickle
 
 from queuelib import queue
 
+from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.utils.reqser import request_to_dict, request_from_dict
 
 
@@ -123,30 +124,60 @@ def _pickle_serialize(obj):
         raise ValueError(str(e)) from e
 
 
-PickleFifoDiskQueueNonRequest = _serializable_queue(
+_PickleFifoSerializationDiskQueue = _serializable_queue(
     _with_mkdir(queue.FifoDiskQueue),
     _pickle_serialize,
     pickle.loads
 )
-PickleLifoDiskQueueNonRequest = _serializable_queue(
+_PickleLifoSerializationDiskQueue = _serializable_queue(
     _with_mkdir(queue.LifoDiskQueue),
     _pickle_serialize,
     pickle.loads
 )
-MarshalFifoDiskQueueNonRequest = _serializable_queue(
+_MarshalFifoSerializationDiskQueue = _serializable_queue(
     _with_mkdir(queue.FifoDiskQueue),
     marshal.dumps,
     marshal.loads
 )
-MarshalLifoDiskQueueNonRequest = _serializable_queue(
+_MarshalLifoSerializationDiskQueue = _serializable_queue(
     _with_mkdir(queue.LifoDiskQueue),
     marshal.dumps,
     marshal.loads
 )
 
-PickleFifoDiskQueue = _scrapy_serialization_queue(PickleFifoDiskQueueNonRequest)
-PickleLifoDiskQueue = _scrapy_serialization_queue(PickleLifoDiskQueueNonRequest)
-MarshalFifoDiskQueue = _scrapy_serialization_queue(MarshalFifoDiskQueueNonRequest)
-MarshalLifoDiskQueue = _scrapy_serialization_queue(MarshalLifoDiskQueueNonRequest)
+# public queue classes
+PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue)
+PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue)
+MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue)
+MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue)
 FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)
 LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)
+
+
+# deprecated queue classes
+_subclass_warn_message = "{cls} inherits from deprecated class {old}"
+_instance_warn_message = "{cls} is deprecated"
+PickleFifoDiskQueueNonRequest = create_deprecated_class(
+    name="PickleFifoDiskQueueNonRequest",
+    new_class=_PickleFifoSerializationDiskQueue,
+    subclass_warn_message=_subclass_warn_message,
+    instance_warn_message=_instance_warn_message,
+)
+PickleLifoDiskQueueNonRequest = create_deprecated_class(
+    name="PickleLifoDiskQueueNonRequest",
+    new_class=_PickleLifoSerializationDiskQueue,
+    subclass_warn_message=_subclass_warn_message,
+    instance_warn_message=_instance_warn_message,
+)
+MarshalFifoDiskQueueNonRequest = create_deprecated_class(
+    name="MarshalFifoDiskQueueNonRequest",
+    new_class=_MarshalFifoSerializationDiskQueue,
+    subclass_warn_message=_subclass_warn_message,
+    instance_warn_message=_instance_warn_message,
+)
+MarshalLifoDiskQueueNonRequest = create_deprecated_class(
+    name="MarshalLifoDiskQueueNonRequest",
+    new_class=_MarshalLifoSerializationDiskQueue,
+    subclass_warn_message=_subclass_warn_message,
+    instance_warn_message=_instance_warn_message,
+)
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index becacce62..acc821b83 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -3,10 +3,10 @@ import sys
 
 from queuelib.tests import test_queue as t
 from scrapy.squeues import (
-    MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue,
-    MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue,
-    PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue,
-    PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue
+    _MarshalFifoSerializationDiskQueue,
+    _MarshalLifoSerializationDiskQueue,
+    _PickleFifoSerializationDiskQueue,
+    _PickleLifoSerializationDiskQueue,
 )
 from scrapy.item import Item, Field
 from scrapy.http import Request
@@ -53,7 +53,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
     chunksize = 100000
 
     def queue(self):
-        return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)
+        return _MarshalFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize)
 
 
 class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
@@ -77,7 +77,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
     chunksize = 100000
 
     def queue(self):
-        return PickleFifoDiskQueue(self.qpath, chunksize=self.chunksize)
+        return _PickleFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize)
 
     def test_serialize_item(self):
         q = self.queue()
@@ -155,13 +155,13 @@ class LifoDiskQueueTestMixin:
 class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
 
     def queue(self):
-        return MarshalLifoDiskQueue(self.qpath)
+        return _MarshalLifoSerializationDiskQueue(self.qpath)
 
 
 class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
 
     def queue(self):
-        return PickleLifoDiskQueue(self.qpath)
+        return _PickleLifoSerializationDiskQueue(self.qpath)
 
     def test_serialize_item(self):
         q = self.queue()

From 4f500342c8ad4674b191e1fab0d1b2ac944d7d3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= <tomas.hrnciar@me.com>
Date: Wed, 28 Apr 2021 11:57:44 +0200
Subject: [PATCH 379/716] Require setuptools, scrapy/cmdline.py, /setup.py and
 tests/test_webclient.py import pkg_resources

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 2b60a10af..b1bb64575 100644
--- a/setup.py
+++ b/setup.py
@@ -32,6 +32,7 @@ install_requires = [
     'protego>=0.1.15',
     'itemadapter>=0.1.0',
     'h2>=3.0,<4.0',
+    'setuptools',
 ]
 extras_require = {}
 cpython_dependencies = [

From 19c7415aae1678631d5ca115a13094b6bd70f245 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 1 May 2021 16:34:39 -0300
Subject: [PATCH 380/716] Request type hints

---
 scrapy/downloadermiddlewares/retry.py |  2 +-
 scrapy/http/request/__init__.py       | 69 ++++++++++++++++-----------
 scrapy/utils/curl.py                  |  2 +-
 3 files changed, 42 insertions(+), 31 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 5965a1c6c..f1fdc3858 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -98,7 +98,7 @@ def get_retry_request(
             {'request': request, 'retry_times': retry_times, 'reason': reason},
             extra={'spider': spider}
         )
-        new_request = request.copy()
+        new_request: Request = request.copy()
         new_request.meta['retry_times'] = retry_times
         new_request.dont_filter = True
         if priority_adjust is None:
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 498f1b052..3cce9f501 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -4,22 +4,38 @@ requests in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+from typing import Callable, List, Optional, Type, TypeVar, Union
+
 from w3lib.url import safe_url_string
 
+from scrapy.http.common import obsolete_setter
 from scrapy.http.headers import Headers
+from scrapy.utils.curl import curl_to_request_kwargs
 from scrapy.utils.python import to_bytes
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
-from scrapy.http.common import obsolete_setter
-from scrapy.utils.curl import curl_to_request_kwargs
+
+
+RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
 
 
 class Request(object_ref):
-
-    def __init__(self, url, callback=None, method='GET', headers=None, body=None,
-                 cookies=None, meta=None, encoding='utf-8', priority=0,
-                 dont_filter=False, errback=None, flags=None, cb_kwargs=None):
-
+    def __init__(
+        self,
+        url: str,
+        callback: Optional[Callable] = None,
+        method: str = "GET",
+        headers: Optional[dict] = None,
+        body: Optional[Union[bytes, str]] = None,
+        cookies: Optional[Union[dict, List[dict]]]=None,
+        meta: Optional[dict] = None,
+        encoding: str = "utf-8",
+        priority: int = 0,
+        dont_filter: bool = False,
+        errback: Optional[Callable] = None,
+        flags: Optional[List[str]] = None,
+        cb_kwargs: Optional[dict] = None,
+    ) -> None:
         self._encoding = encoding  # this one has to be set first
         self.method = str(method).upper()
         self._set_url(url)
@@ -44,23 +60,23 @@ class Request(object_ref):
         self.flags = [] if flags is None else list(flags)
 
     @property
-    def cb_kwargs(self):
+    def cb_kwargs(self) -> dict:
         if self._cb_kwargs is None:
             self._cb_kwargs = {}
         return self._cb_kwargs
 
     @property
-    def meta(self):
+    def meta(self) -> dict:
         if self._meta is None:
             self._meta = {}
         return self._meta
 
-    def _get_url(self):
+    def _get_url(self) -> str:
         return self._url
 
-    def _set_url(self, url):
+    def _set_url(self, url: str) -> None:
         if not isinstance(url, str):
-            raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}')
+            raise TypeError(f"Request url must be str, got {type(url).__name__}")
 
         s = safe_url_string(url, self.encoding)
         self._url = escape_ajax(s)
@@ -74,34 +90,28 @@ class Request(object_ref):
 
     url = property(_get_url, obsolete_setter(_set_url, 'url'))
 
-    def _get_body(self):
+    def _get_body(self) -> bytes:
         return self._body
 
-    def _set_body(self, body):
-        if body is None:
-            self._body = b''
-        else:
-            self._body = to_bytes(body, self.encoding)
+    def _set_body(self, body: Optional[Union[str, bytes]]) -> None:
+        self._body = b"" if body is None else to_bytes(body, self.encoding)
 
     body = property(_get_body, obsolete_setter(_set_body, 'body'))
 
     @property
-    def encoding(self):
+    def encoding(self) -> str:
         return self._encoding
 
-    def __str__(self):
+    def __str__(self) -> str:
         return f"<{self.method} {self.url}>"
 
     __repr__ = __str__
 
-    def copy(self):
-        """Return a copy of this Request"""
+    def copy(self) -> RequestTypeVar:
         return self.replace()
 
-    def replace(self, *args, **kwargs):
-        """Create a new Request with the same attributes except for those
-        given new values.
-        """
+    def replace(self, *args, **kwargs) -> RequestTypeVar:
+        """Create a new Request with the same attributes except for those given new values"""
         for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
                   'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:
             kwargs.setdefault(x, getattr(self, x))
@@ -109,7 +119,9 @@ class Request(object_ref):
         return cls(*args, **kwargs)
 
     @classmethod
-    def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs):
+    def from_curl(
+        cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, **kwargs
+    ) -> RequestTypeVar:
         """Create a Request object from a string containing a `cURL
         <https://curl.haxx.se/>`_ command. It populates the HTTP method, the
         URL, the headers, the cookies and the body. It accepts the same
@@ -136,8 +148,7 @@ class Request(object_ref):
 
         To translate a cURL command into a Scrapy request,
         you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
-
-       """
+        """
         request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
         request_kwargs.update(kwargs)
         return cls(**request_kwargs)
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index d8b3deaa1..74f82ad75 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -54,7 +54,7 @@ def _parse_headers_and_cookies(parsed_args):
     return headers, cookies
 
 
-def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
+def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = True) -> dict:
     """Convert a cURL command syntax to Request kwargs.
 
     :param str curl_command: string containing the curl command

From 34b216289c31c27ad6256ff95382505a9d84adb3 Mon Sep 17 00:00:00 2001
From: Renne Rocha <renne@rennerocha.com>
Date: Thu, 6 May 2021 11:34:05 -0300
Subject: [PATCH 381/716] Update link for reasoning value of URLLENGTH_LIMIT
 (#5134)

---
 docs/topics/settings.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index e4fb2baf7..2506497e2 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1619,7 +1619,7 @@ Default: ``2083``
 Scope: ``spidermiddlewares.urllength``
 
 The maximum URL length to allow for crawled URLs. For more information about
-the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html
+the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
 
 .. setting:: USER_AGENT
 
@@ -1642,7 +1642,6 @@ case to see how to enable and use them.
 
 .. settingslist::
 
-
 .. _Amazon web services: https://aws.amazon.com/
 .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
 .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search

From cec36a9284641bb7b69a2081ad92cd7d4ee25934 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Mon, 10 May 2021 13:00:08 -0300
Subject: [PATCH 382/716] Refactor request to/from dict (#5130)

---
 docs/topics/request-response.rst              |  17 ++-
 scrapy/http/request/__init__.py               |  70 ++++++++++--
 scrapy/http/request/json_request.py           |   8 ++
 scrapy/squeues.py                             |   8 +-
 scrapy/utils/reqser.py                        | 103 +++---------------
 scrapy/utils/request.py                       |  27 ++++-
 ...t_utils_reqser.py => test_request_dict.py} |  76 +++++++++----
 7 files changed, 183 insertions(+), 126 deletions(-)
 rename tests/{test_utils_reqser.py => test_request_dict.py} (68%)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 500781c05..73b5a858f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -26,10 +26,6 @@ Request objects
 
 .. autoclass:: Request
 
-    A :class:`Request` object represents an HTTP request, which is usually
-    generated in the Spider and executed by the Downloader, and thus generating
-    a :class:`Response`.
-
     :param url: the URL of this request
 
         If the URL is invalid, a :exc:`ValueError` exception is raised.
@@ -205,6 +201,8 @@ Request objects
         ``failure.request.cb_kwargs`` in the request's errback. For more information,
         see :ref:`errback-cb_kwargs`.
 
+    .. autoattribute:: Request.attributes
+
     .. method:: Request.copy()
 
        Return a new Request which is a copy of this Request. See also:
@@ -220,6 +218,15 @@ Request objects
 
     .. automethod:: from_curl
 
+    .. automethod:: to_dict
+
+
+Other functions related to requests
+-----------------------------------
+
+.. autofunction:: scrapy.utils.request.request_from_dict
+
+
 .. _topics-request-response-ref-request-callback-arguments:
 
 Passing additional data to callback functions
@@ -642,6 +649,8 @@ dealing with JSON requests.
        data into JSON format.
    :type dumps_kwargs: dict
 
+   .. autoattribute:: JsonRequest.attributes
+
 JsonRequest usage example
 -------------------------
 
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 498f1b052..ad884feac 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -4,17 +4,37 @@ requests in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+import inspect
+from typing import Optional, Tuple
+
 from w3lib.url import safe_url_string
 
+import scrapy
+from scrapy.http.common import obsolete_setter
 from scrapy.http.headers import Headers
+from scrapy.utils.curl import curl_to_request_kwargs
 from scrapy.utils.python import to_bytes
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
-from scrapy.http.common import obsolete_setter
-from scrapy.utils.curl import curl_to_request_kwargs
 
 
 class Request(object_ref):
+    """Represents an HTTP request, which is usually generated in a Spider and
+    executed by the Downloader, thus generating a :class:`Response`.
+    """
+
+    attributes: Tuple[str, ...] = (
+        "url", "callback", "method", "headers", "body",
+        "cookies", "meta", "encoding", "priority",
+        "dont_filter", "errback", "flags", "cb_kwargs",
+    )
+    """A tuple of :class:`str` objects containing the name of all public
+    attributes of the class that are also keyword parameters of the
+    ``__init__`` method.
+
+    Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and
+    :func:`~scrapy.utils.request.request_from_dict`.
+    """
 
     def __init__(self, url, callback=None, method='GET', headers=None, body=None,
                  cookies=None, meta=None, encoding='utf-8', priority=0,
@@ -99,11 +119,8 @@ class Request(object_ref):
         return self.replace()
 
     def replace(self, *args, **kwargs):
-        """Create a new Request with the same attributes except for those
-        given new values.
-        """
-        for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
-                  'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:
+        """Create a new Request with the same attributes except for those given new values"""
+        for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)
@@ -136,8 +153,43 @@ class Request(object_ref):
 
         To translate a cURL command into a Scrapy request,
         you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
-
-       """
+        """
         request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
         request_kwargs.update(kwargs)
         return cls(**request_kwargs)
+
+    def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict:
+        """Return a dictionary containing the Request's data.
+
+        Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
+
+        If a spider is given, this method will try to find out the name of the spider methods used as callback
+        and errback and include them in the output dict, raising an exception if they cannot be found.
+        """
+        d = {
+            "url": self.url,  # urls are safe (safe_string_url)
+            "callback": _find_method(spider, self.callback) if callable(self.callback) else self.callback,
+            "errback": _find_method(spider, self.errback) if callable(self.errback) else self.errback,
+            "headers": dict(self.headers),
+        }
+        for attr in self.attributes:
+            d.setdefault(attr, getattr(self, attr))
+        if type(self) is not Request:
+            d["_class"] = self.__module__ + '.' + self.__class__.__name__
+        return d
+
+
+def _find_method(obj, func):
+    """Helper function for Request.to_dict"""
+    # Only instance methods contain ``__func__``
+    if obj and hasattr(func, '__func__'):
+        members = inspect.getmembers(obj, predicate=inspect.ismethod)
+        for name, obj_func in members:
+            # We need to use __func__ to access the original function object because instance
+            # method objects are generated each time attribute is retrieved from instance.
+            #
+            # Reference: The standard type hierarchy
+            # https://docs.python.org/3/reference/datamodel.html
+            if obj_func.__func__ is func.__func__:
+                return name
+    raise ValueError(f"Function {func} is not an instance method in: {obj}")
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index eae3f9f6b..04e80d897 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -8,12 +8,16 @@ See documentation in docs/topics/request-response.rst
 import copy
 import json
 import warnings
+from typing import Tuple
 
 from scrapy.http.request import Request
 from scrapy.utils.deprecate import create_deprecated_class
 
 
 class JsonRequest(Request):
+
+    attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
+
     def __init__(self, *args, **kwargs):
         dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {}))
         dumps_kwargs.setdefault('sort_keys', True)
@@ -36,6 +40,10 @@ class JsonRequest(Request):
         self.headers.setdefault('Content-Type', 'application/json')
         self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
 
+    @property
+    def dumps_kwargs(self):
+        return self._dumps_kwargs
+
     def replace(self, *args, **kwargs):
         body_passed = kwargs.get('body', None) is not None
         data = kwargs.pop('data', None)
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 16f7bf4b6..dff9b1350 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -9,7 +9,7 @@ import pickle
 from queuelib import queue
 
 from scrapy.utils.deprecate import create_deprecated_class
-from scrapy.utils.reqser import request_to_dict, request_from_dict
+from scrapy.utils.request import request_from_dict
 
 
 def _with_mkdir(queue_class):
@@ -68,14 +68,14 @@ def _scrapy_serialization_queue(queue_class):
             return cls(crawler, key)
 
         def push(self, request):
-            request = request_to_dict(request, self.spider)
+            request = request.to_dict(spider=self.spider)
             return super().push(request)
 
         def pop(self):
             request = super().pop()
             if not request:
                 return None
-            return request_from_dict(request, self.spider)
+            return request_from_dict(request, spider=self.spider)
 
         def peek(self):
             """Returns the next object to be returned by :meth:`pop`,
@@ -87,7 +87,7 @@ def _scrapy_serialization_queue(queue_class):
             request = super().peek()
             if not request:
                 return None
-            return request_from_dict(request, self.spider)
+            return request_from_dict(request, spider=self.spider)
 
     return ScrapyRequestQueue
 
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index d38b1bc4d..c254b9f82 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -1,95 +1,22 @@
-"""
-Helper functions for serializing (and deserializing) requests.
-"""
-import inspect
+import warnings
+from typing import Optional
 
-from scrapy.http import Request
-from scrapy.utils.python import to_unicode
-from scrapy.utils.misc import load_object
+import scrapy
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.request import request_from_dict as _from_dict
 
 
-def request_to_dict(request, spider=None):
-    """Convert Request object to a dict.
-
-    If a spider is given, it will try to find out the name of the spider method
-    used in the callback and store that as the callback.
-    """
-    cb = request.callback
-    if callable(cb):
-        cb = _find_method(spider, cb)
-    eb = request.errback
-    if callable(eb):
-        eb = _find_method(spider, eb)
-    d = {
-        'url': to_unicode(request.url),  # urls should be safe (safe_string_url)
-        'callback': cb,
-        'errback': eb,
-        'method': request.method,
-        'headers': dict(request.headers),
-        'body': request.body,
-        'cookies': request.cookies,
-        'meta': request.meta,
-        '_encoding': request._encoding,
-        'priority': request.priority,
-        'dont_filter': request.dont_filter,
-        'flags': request.flags,
-        'cb_kwargs': request.cb_kwargs,
-    }
-    if type(request) is not Request:
-        d['_class'] = request.__module__ + '.' + request.__class__.__name__
-    return d
+warnings.warn(
+    ("Module scrapy.utils.reqser is deprecated, please use request.to_dict method"
+     " and/or scrapy.utils.request.request_from_dict instead"),
+    category=ScrapyDeprecationWarning,
+    stacklevel=2,
+)
 
 
-def request_from_dict(d, spider=None):
-    """Create Request object from a dict.
-
-    If a spider is given, it will try to resolve the callbacks looking at the
-    spider for methods with the same name.
-    """
-    cb = d['callback']
-    if cb and spider:
-        cb = _get_method(spider, cb)
-    eb = d['errback']
-    if eb and spider:
-        eb = _get_method(spider, eb)
-    request_cls = load_object(d['_class']) if '_class' in d else Request
-    return request_cls(
-        url=to_unicode(d['url']),
-        callback=cb,
-        errback=eb,
-        method=d['method'],
-        headers=d['headers'],
-        body=d['body'],
-        cookies=d['cookies'],
-        meta=d['meta'],
-        encoding=d['_encoding'],
-        priority=d['priority'],
-        dont_filter=d['dont_filter'],
-        flags=d.get('flags'),
-        cb_kwargs=d.get('cb_kwargs'),
-    )
+def request_to_dict(request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None) -> dict:
+    return request.to_dict(spider=spider)
 
 
-def _find_method(obj, func):
-    # Only instance methods contain ``__func__``
-    if obj and hasattr(func, '__func__'):
-        members = inspect.getmembers(obj, predicate=inspect.ismethod)
-        for name, obj_func in members:
-            # We need to use __func__ to access the original
-            # function object because instance method objects
-            # are generated each time attribute is retrieved from
-            # instance.
-            #
-            # Reference: The standard type hierarchy
-            # https://docs.python.org/3/reference/datamodel.html
-            if obj_func.__func__ is func.__func__:
-                return name
-    raise ValueError(f"Function {func} is not an instance method in: {obj}")
-
-
-def _get_method(obj, name):
-    name = str(name)
-    try:
-        return getattr(obj, name)
-    except AttributeError:
-        raise ValueError(f"Method {name!r} not found in: {obj}")
+def request_from_dict(d: dict, spider: Optional["scrapy.Spider"] = None) -> "scrapy.Request":
+    return _from_dict(d, spider=spider)
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 541368423..57dcc5f2c 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -11,8 +11,9 @@ from weakref import WeakKeyDictionary
 from w3lib.http import basic_auth_header
 from w3lib.url import canonicalize_url
 
-from scrapy.http import Request
+from scrapy import Request, Spider
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode
 
 
@@ -106,3 +107,27 @@ def referer_str(request: Request) -> Optional[str]:
     if referrer is None:
         return referrer
     return to_unicode(referrer, errors='replace')
+
+
+def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
+    """Create a :class:`~scrapy.Request` object from a dict.
+
+    If a spider is given, it will try to resolve the callbacks looking at the
+    spider for methods with the same name.
+    """
+    request_cls = load_object(d["_class"]) if "_class" in d else Request
+    kwargs = {key: value for key, value in d.items() if key in request_cls.attributes}
+    if d.get("callback") and spider:
+        kwargs["callback"] = _get_method(spider, d["callback"])
+    if d.get("errback") and spider:
+        kwargs["errback"] = _get_method(spider, d["errback"])
+    return request_cls(**kwargs)
+
+
+def _get_method(obj, name):
+    """Helper function for request_from_dict"""
+    name = str(name)
+    try:
+        return getattr(obj, name)
+    except AttributeError:
+        raise ValueError(f"Method {name!r} not found in: {obj}")
diff --git a/tests/test_utils_reqser.py b/tests/test_request_dict.py
similarity index 68%
rename from tests/test_utils_reqser.py
rename to tests/test_request_dict.py
index ee68cf6b1..5bdcb975b 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_request_dict.py
@@ -1,8 +1,16 @@
+import sys
 import unittest
+import warnings
+from contextlib import suppress
 
-from scrapy.http import Request, FormRequest
-from scrapy.spiders import Spider
-from scrapy.utils.reqser import request_to_dict, request_from_dict
+from scrapy import Spider, Request
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.http import FormRequest, JsonRequest
+from scrapy.utils.request import request_from_dict
+
+
+class CustomRequest(Request):
+    pass
 
 
 class RequestSerializationTest(unittest.TestCase):
@@ -27,7 +35,8 @@ class RequestSerializationTest(unittest.TestCase):
             priority=20,
             meta={'a': 'b'},
             cb_kwargs={'k': 'v'},
-            flags=['testFlag'])
+            flags=['testFlag'],
+        )
         self._assert_serializes_ok(r, spider=self.spider)
 
     def test_latin1_body(self):
@@ -39,7 +48,7 @@ class RequestSerializationTest(unittest.TestCase):
         self._assert_serializes_ok(r)
 
     def _assert_serializes_ok(self, request, spider=None):
-        d = request_to_dict(request, spider=spider)
+        d = request.to_dict(spider=spider)
         request2 = request_from_dict(d, spider=spider)
         self._assert_same_request(request, request2)
 
@@ -54,16 +63,21 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertEqual(r1.cookies, r2.cookies)
         self.assertEqual(r1.meta, r2.meta)
         self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
+        self.assertEqual(r1.encoding, r2.encoding)
         self.assertEqual(r1._encoding, r2._encoding)
         self.assertEqual(r1.priority, r2.priority)
         self.assertEqual(r1.dont_filter, r2.dont_filter)
         self.assertEqual(r1.flags, r2.flags)
+        if isinstance(r1, JsonRequest):
+            self.assertEqual(r1.dumps_kwargs, r2.dumps_kwargs)
 
     def test_request_class(self):
-        r = FormRequest("http://www.example.com")
-        self._assert_serializes_ok(r, spider=self.spider)
-        r = CustomRequest("http://www.example.com")
-        self._assert_serializes_ok(r, spider=self.spider)
+        r1 = FormRequest("http://www.example.com")
+        self._assert_serializes_ok(r1, spider=self.spider)
+        r2 = CustomRequest("http://www.example.com")
+        self._assert_serializes_ok(r2, spider=self.spider)
+        r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
+        self._assert_serializes_ok(r3, spider=self.spider)
 
     def test_callback_serialization(self):
         r = Request("http://www.example.com", callback=self.spider.parse_item,
@@ -75,7 +89,7 @@ class RequestSerializationTest(unittest.TestCase):
                     callback=self.spider.parse_item_reference,
                     errback=self.spider.handle_error_reference)
         self._assert_serializes_ok(r, spider=self.spider)
-        request_dict = request_to_dict(r, self.spider)
+        request_dict = r.to_dict(spider=self.spider)
         self.assertEqual(request_dict['callback'], 'parse_item_reference')
         self.assertEqual(request_dict['errback'], 'handle_error_reference')
 
@@ -84,7 +98,7 @@ class RequestSerializationTest(unittest.TestCase):
                     callback=self.spider._TestSpider__parse_item_reference,
                     errback=self.spider._TestSpider__handle_error_reference)
         self._assert_serializes_ok(r, spider=self.spider)
-        request_dict = request_to_dict(r, self.spider)
+        request_dict = r.to_dict(spider=self.spider)
         self.assertEqual(request_dict['callback'],
                          '_TestSpider__parse_item_reference')
         self.assertEqual(request_dict['errback'],
@@ -110,18 +124,16 @@ class RequestSerializationTest(unittest.TestCase):
 
     def test_unserializable_callback1(self):
         r = Request("http://www.example.com", callback=lambda x: x)
-        self.assertRaises(ValueError, request_to_dict, r)
-        self.assertRaises(ValueError, request_to_dict, r, spider=self.spider)
+        self.assertRaises(ValueError, r.to_dict, spider=self.spider)
 
     def test_unserializable_callback2(self):
         r = Request("http://www.example.com", callback=self.spider.parse_item)
-        self.assertRaises(ValueError, request_to_dict, r)
+        self.assertRaises(ValueError, r.to_dict, spider=None)
 
     def test_unserializable_callback3(self):
         """Parser method is removed or replaced dynamically."""
 
         class MySpider(Spider):
-
             name = 'my_spider'
 
             def parse(self, response):
@@ -130,7 +142,35 @@ class RequestSerializationTest(unittest.TestCase):
         spider = MySpider()
         r = Request("http://www.example.com", callback=spider.parse)
         setattr(spider, 'parse', None)
-        self.assertRaises(ValueError, request_to_dict, r, spider=spider)
+        self.assertRaises(ValueError, r.to_dict, spider=spider)
+
+    def test_callback_not_available(self):
+        """Callback method is not available in the spider passed to from_dict"""
+        spider = TestSpiderDelegation()
+        r = Request("http://www.example.com", callback=spider.delegated_callback)
+        d = r.to_dict(spider=spider)
+        self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo"))
+
+
+class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest):
+    def _assert_serializes_ok(self, request, spider=None):
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always")
+            with suppress(KeyError):
+                del sys.modules["scrapy.utils.reqser"]  # delete module to reset the deprecation warning
+
+            from scrapy.utils.reqser import request_from_dict as _from_dict, request_to_dict as _to_dict
+
+            request_copy = _from_dict(_to_dict(request, spider), spider)
+            self._assert_same_request(request, request_copy)
+
+            self.assertEqual(len(caught), 1)
+            self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning))
+            self.assertEqual(
+                "Module scrapy.utils.reqser is deprecated, please use request.to_dict method"
+                " and/or scrapy.utils.request.request_from_dict instead",
+                str(caught[0].message),
+            )
 
 
 class TestSpiderMixin:
@@ -177,7 +217,3 @@ class TestSpider(Spider, TestSpiderMixin):
 
     def __parse_item_private(self, response):
         pass
-
-
-class CustomRequest(Request):
-    pass

From bd60c3f41fd25e7b5a413cf5c112166b813c2691 Mon Sep 17 00:00:00 2001
From: Shinichi Takayanagi <shinichi.takayanagi@gmail.com>
Date: Tue, 11 May 2021 04:58:04 +0900
Subject: [PATCH 383/716] More documentation for setting spider atributes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* docs: require sphinx-rtd-theme>=0.5.2 and the latest pip to prevent installing breaking docutils>=0.17

* Update feed-exports.rst

* Update feed-exports.rst

* Reflects the comments

* Remove redundant newline

* Update docs/topics/feed-exports.rst

Co-authored-by: Adrián Chaves <adrian@chaves.io>

* Apply suggestions from code review

Co-authored-by: Adrián Chaves <adrian@chaves.io>

Co-authored-by: Adrián Chaves <adrian@chaves.io>
Co-authored-by: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
---
 docs/topics/feed-exports.rst | 3 +++
 docs/topics/spiders.rst      | 8 ++++++++
 2 files changed, 11 insertions(+)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index e772a461c..26c247cdd 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -135,6 +135,9 @@ Here are some examples to illustrate:
 
     -   ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
 
+.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence 
+          they can also be used as storage URI parameters.
+
 
 .. _topics-feed-storage-backends:
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 2056664c7..a3e9f410f 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -294,6 +294,14 @@ The above example can also be written as follows::
         def start_requests(self):
             yield scrapy.Request(f'http://www.example.com/categories/{self.category}')
 
+If you are :ref:`running Scrapy from a script <run-from-script>`, you can 
+specify spider arguments when calling 
+:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
+:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`::
+
+    process = CrawlerProcess()
+    process.crawl(MySpider, category="electronics")
+
 Keep in mind that spider arguments are only strings.
 The spider will not do any parsing on its own.
 If you were to set the ``start_urls`` attribute from the command line,

From c5b1ee810167266fcd259f263dbfc0fe0204761a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 11 May 2021 09:04:53 +0200
Subject: [PATCH 384/716] Make Twisted[http2] installation optional (#5113)

Co-authored-by: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
---
 conftest.py                             |  9 +++++++++
 docs/topics/settings.rst                | 14 ++++++++-----
 setup.py                                |  3 +--
 tests/test_downloader_handlers_http2.py | 26 ++++++++++++++++++++-----
 tests/test_http2_client_protocol.py     | 13 ++++++++-----
 tox.ini                                 | 11 +++++------
 6 files changed, 53 insertions(+), 23 deletions(-)

diff --git a/conftest.py b/conftest.py
index e4dd80de0..4931c5a79 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,6 +1,7 @@
 from pathlib import Path
 
 import pytest
+from twisted.web.http import H2_ENABLED
 
 from scrapy.utils.reactor import install_reactor
 
@@ -25,6 +26,14 @@ for line in open('tests/ignores.txt'):
     if file_path and file_path[0] != '#':
         collect_ignore.append(file_path)
 
+if not H2_ENABLED:
+    collect_ignore.extend(
+        (
+            'scrapy/core/downloader/handlers/http2.py',
+            *_py_files("scrapy/core/http2"),
+        )
+    )
+
 
 @pytest.fixture()
 def chdir(tmpdir):
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 2506497e2..0b290598f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -680,12 +680,16 @@ handler (without replacement), place this in your ``settings.py``::
 
 .. _http2:
 
-The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
-:setting:`DOWNLOAD_HANDLERS` as follows::
+The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
 
-    DOWNLOAD_HANDLERS = {
-        'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
-    }
+#.  Install ``Twisted[http2]>=17.9.0`` to install the packages required to
+    enable HTTP/2 support in Twisted.
+
+#.  Update :setting:`DOWNLOAD_HANDLERS` as follows::
+
+        DOWNLOAD_HANDLERS = {
+            'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
+        }
 
 .. warning::
 
diff --git a/setup.py b/setup.py
index b1bb64575..ed2b6e347 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support():
 
 
 install_requires = [
-    'Twisted[http2]>=17.9.0',
+    'Twisted>=17.9.0',
     'cryptography>=2.0',
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
@@ -31,7 +31,6 @@ install_requires = [
     'zope.interface>=4.1.3',
     'protego>=0.1.15',
     'itemadapter>=0.1.0',
-    'h2>=3.0,<4.0',
     'setuptools',
 ]
 extras_require = {}
diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index 439778014..53bb4fe92 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -1,5 +1,5 @@
 import json
-from unittest import mock
+from unittest import mock, skipIf
 
 from pytest import mark
 from testfixtures import LogCapture
@@ -7,8 +7,8 @@ from twisted.internet import defer, error, reactor
 from twisted.trial import unittest
 from twisted.web import server
 from twisted.web.error import SchemeNotSupported
+from twisted.web.http import H2_ENABLED
 
-from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.misc import create_instance
@@ -21,11 +21,17 @@ from tests.test_downloader_handlers import (
 )
 
 
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2TestCase(Https11TestCase):
+
     scheme = 'https'
-    download_handler_cls = H2DownloadHandler
     HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
 
+    @classmethod
+    def setUpClass(cls):
+        from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+        cls.download_handler_cls = H2DownloadHandler
+
     def test_protocol(self):
         request = Request(self.getURL("host"), method="GET")
         d = self.download_request(request, Spider("foo"))
@@ -187,9 +193,14 @@ class Https2InvalidDNSPattern(Https2TestCase):
         super(Https2InvalidDNSPattern, self).setUp()
 
 
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2CustomCiphers(Https11CustomCiphers):
     scheme = 'https'
-    download_handler_cls = H2DownloadHandler
+
+    @classmethod
+    def setUpClass(cls):
+        from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+        cls.download_handler_cls = H2DownloadHandler
 
 
 class Http2MockServerTestCase(Http11MockServerTestCase):
@@ -201,6 +212,7 @@ class Http2MockServerTestCase(Http11MockServerTestCase):
     }
 
 
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2ProxyTestCase(Http11ProxyTestCase):
     # only used for HTTPS tests
     keyfile = 'keys/localhost.key'
@@ -209,9 +221,13 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
     scheme = 'https'
     host = u'127.0.0.1'
 
-    download_handler_cls = H2DownloadHandler
     expected_http_proxy_request_body = b'/'
 
+    @classmethod
+    def setUpClass(cls):
+        from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+        cls.download_handler_cls = H2DownloadHandler
+
     def setUp(self):
         site = server.Site(UriResource(), timeout=None)
         self.port = reactor.listenSSL(
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 8b2f6a11d..677ede92b 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -5,10 +5,9 @@ import re
 import shutil
 import string
 from ipaddress import IPv4Address
-from unittest import mock
+from unittest import mock, skipIf
 from urllib.parse import urlencode
 
-from h2.exceptions import InvalidBodyLengthError
 from twisted.internet import reactor
 from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
@@ -17,12 +16,10 @@ from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certif
 from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
 from twisted.web.client import ResponseFailed, URI
-from twisted.web.http import Request as TxRequest
+from twisted.web.http import H2_ENABLED, Request as TxRequest
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.static import File
 
-from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
-from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname
 from scrapy.http import Request, Response, JsonRequest
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
@@ -173,6 +170,7 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
     return PrivateCertificate.loadPEM(pem)
 
 
+@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
 class Https2ClientProtocolTestCase(TestCase):
     scheme = 'https'
     key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key')
@@ -220,6 +218,7 @@ class Https2ClientProtocolTestCase(TestCase):
         uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8'))
 
         self.conn_closed_deferred = Deferred()
+        from scrapy.core.http2.protocol import H2ClientFactory
         h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)
         client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options)
         self.client = yield client_endpoint.connect(h2_client_factory)
@@ -426,6 +425,7 @@ class Https2ClientProtocolTestCase(TestCase):
 
         def assert_failure(failure: Failure):
             self.assertTrue(len(failure.value.reasons) > 0)
+            from h2.exceptions import InvalidBodyLengthError
             self.assertTrue(any(
                 isinstance(error, InvalidBodyLengthError)
                 for error in failure.value.reasons
@@ -511,6 +511,7 @@ class Https2ClientProtocolTestCase(TestCase):
 
         def assert_inactive_stream(failure):
             self.assertIsNotNone(failure.check(ResponseFailed))
+            from scrapy.core.http2.stream import InactiveStreamClosed
             self.assertTrue(any(
                 isinstance(e, InactiveStreamClosed)
                 for e in failure.value.reasons
@@ -596,6 +597,7 @@ class Https2ClientProtocolTestCase(TestCase):
         request = Request(url)
 
         def assert_invalid_hostname(failure: Failure):
+            from scrapy.core.http2.stream import InvalidHostname
             self.assertIsNotNone(failure.check(InvalidHostname))
             error_msg = str(failure.value)
             self.assertIn('localhost', error_msg)
@@ -633,6 +635,7 @@ class Https2ClientProtocolTestCase(TestCase):
 
         def assert_timeout_error(failure: Failure):
             for err in failure.value.reasons:
+                from scrapy.core.http2.protocol import H2ClientProtocol
                 if isinstance(err, TimeoutError):
                     self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err))
                     break
diff --git a/tox.ini b/tox.ini
index 5b0606f8f..8167aff96 100644
--- a/tox.ini
+++ b/tox.ini
@@ -50,6 +50,8 @@ commands =
 basepython = python3
 deps =
     {[testenv]deps}
+    # Twisted[http2] is required to import some files
+    Twisted[http2]>=17.9.0
     pytest-flake8
 commands =
     py.test --flake8 {posargs:docs scrapy tests}
@@ -57,12 +59,7 @@ commands =
 [testenv:pylint]
 basepython = python3
 deps =
-    {[testenv]deps}
-    # Optional dependencies
-    boto
-    reppy
-    robotexclusionrulesparser
-    # Test dependencies
+    {[testenv:extra-deps]deps}
     pylint
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
@@ -119,9 +116,11 @@ setenv =
 [testenv:extra-deps]
 deps =
     {[testenv]deps}
+    boto
     reppy
     robotexclusionrulesparser
     Pillow>=4.0.0
+    Twisted[http2]>=17.9.0
 
 [testenv:asyncio]
 commands =

From ee682af3b06d48815dbdaa27c1177b94aaf679e1 Mon Sep 17 00:00:00 2001
From: Bhavesh <35660861+Bhavesh0327@users.noreply.github.com>
Date: Wed, 12 May 2021 01:53:02 +0530
Subject: [PATCH 385/716] [Fix] Change the truncation limit of Proxy
 TunnelError from 32 to 1000 (#5007)

* [Fix] Change the truncation limit oof Proxy TunnelError from 32 to 64

* [Fix] Change the truncation limit for Proxy tunnel error

* [Fix] flake8 check

* [Fix] formatting issues

* [Remove] coverage report

* [Fix] truncation error issue

* [Fix] formatting issues

* [Remove] coverage report
---
 scrapy/core/downloader/handlers/http11.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 25cb3ec62..073f35891 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -98,8 +98,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
     with this endpoint comes from the pool and a CONNECT has already been issued
     for it.
     """
-
-    _responseMatcher = re.compile(br'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
+    _truncatedLength = 1000
+    _responseAnswer = r'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,' + str(_truncatedLength) + r'})'
+    _responseMatcher = re.compile(_responseAnswer.encode())
 
     def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
         proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
@@ -144,7 +145,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
                 extra = {'status': int(respm.group('status')),
                          'reason': respm.group('reason').strip()}
             else:
-                extra = rcvd_bytes[:32]
+                extra = rcvd_bytes[:self._truncatedLength]
             self._tunnelReadyDeferred.errback(
                 TunnelError('Could not open CONNECT tunnel with proxy '
                             f'{self._host}:{self._port} [{extra!r}]')

From 23cfdb058e80a18ee2b66e1b966355f3aca426d0 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Fri, 28 May 2021 09:45:06 +0000
Subject: [PATCH 386/716] Reducing amount of warnings during test run (#5162)

* put flake8 options into separate file to remove pytest warnings

* remove ResourceLeaked warning in pypy

* suppress warnings from twisted

* ignore deprecation warnings here

* ignore deprecation warning in tests of deprecated methods

* ignore deprecation warnings here

* update test classes

* don`t use deprecated method call

* ignore deprecation warnings here

* proper warning class

* more selective ignoring

* Revert "don`t use deprecated method call"

This reverts commit 59216ab5603c4b47574382768614ef4c39d36747.
---
 .flake8                       | 19 +++++++++++++++++++
 .gitignore                    |  2 ++
 conftest.py                   |  9 +++++----
 pytest.ini                    | 19 ++-----------------
 tests/test_exporters.py       | 12 ++++++++----
 tests/test_feedexport.py      |  4 ++--
 tests/test_http_response.py   | 12 ++++++++----
 tests/test_item.py            | 22 ++++++++++++----------
 tests/test_utils_deprecate.py |  2 +-
 tests/test_utils_python.py    |  9 +++++++--
 10 files changed, 66 insertions(+), 44 deletions(-)
 create mode 100644 .flake8

diff --git a/.flake8 b/.flake8
new file mode 100644
index 000000000..1c503fb0b
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,19 @@
+[flake8]
+
+max-line-length = 119
+ignore = W503
+
+exclude =
+# Exclude files that are meant to provide top-level imports
+# E402: Module level import not at top of file
+# F401: Module imported but unused
+    scrapy/__init__.py E402
+    scrapy/core/downloader/handlers/http.py F401
+    scrapy/http/__init__.py F401
+    scrapy/linkextractors/__init__.py E402 F401
+    scrapy/selector/__init__.py F401
+    scrapy/spiders/__init__.py E402 F401
+
+    # Issues pending a review:
+    scrapy/utils/url.py F403 F405
+    tests/test_loader.py E741
diff --git a/.gitignore b/.gitignore
index 795e2605e..d77d24624 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,8 @@ htmlcov/
 .coverage
 .pytest_cache/
 .coverage.*
+coverage.*
+test-output.*
 .cache/
 .mypy_cache/
 /tests/keys/localhost.crt
diff --git a/conftest.py b/conftest.py
index 4931c5a79..05b4ccdad 100644
--- a/conftest.py
+++ b/conftest.py
@@ -21,10 +21,11 @@ collect_ignore = [
     *_py_files("tests/CrawlerRunner"),
 ]
 
-for line in open('tests/ignores.txt'):
-    file_path = line.strip()
-    if file_path and file_path[0] != '#':
-        collect_ignore.append(file_path)
+with open('tests/ignores.txt') as reader:
+    for line in reader:
+        file_path = line.strip()
+        if file_path and file_path[0] != '#':
+            collect_ignore.append(file_path)
 
 if not H2_ENABLED:
     collect_ignore.extend(
diff --git a/pytest.ini b/pytest.ini
index 0aae09ff5..6de08c78d 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -20,20 +20,5 @@ addopts =
     --ignore=docs/utils
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
-flake8-max-line-length = 119
-flake8-ignore =
-    W503
-
-    # Exclude files that are meant to provide top-level imports
-    # E402: Module level import not at top of file
-    # F401: Module imported but unused
-    scrapy/__init__.py E402
-    scrapy/core/downloader/handlers/http.py F401
-    scrapy/http/__init__.py F401
-    scrapy/linkextractors/__init__.py E402 F401
-    scrapy/selector/__init__.py F401
-    scrapy/spiders/__init__.py E402 F401
-
-    # Issues pending a review:
-    scrapy/utils/url.py F403 F405
-    tests/test_loader.py E741
+filterwarnings=
+    ignore::DeprecationWarning:twisted.web.test.test_webclient
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index ebc477e74..04bae31d3 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -6,12 +6,14 @@ import tempfile
 import unittest
 from io import BytesIO
 from datetime import datetime
+from warnings import catch_warnings, filterwarnings
 
 import lxml.etree
 from itemadapter import ItemAdapter
 
 from scrapy.item import Item, Field
 from scrapy.utils.python import to_unicode
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.exporters import (
     BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter,
     XmlItemExporter, JsonLinesItemExporter, JsonItemExporter,
@@ -172,10 +174,12 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(type(exported['age'][0]['age'][0]), dict)
 
     def test_export_binary(self):
-        exporter = PythonItemExporter(binary=True)
-        value = self.item_class(name='John\xa3', age='22')
-        expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
-        self.assertEqual(expected, exporter.export_item(value))
+        with catch_warnings():
+            filterwarnings('ignore', category=ScrapyDeprecationWarning)
+            exporter = PythonItemExporter(binary=True)
+            value = self.item_class(name='John\xa3', age='22')
+            expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
+            self.assertEqual(expected, exporter.export_item(value))
 
     def test_nonstring_types_item(self):
         item = self._get_nonstring_types_item()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d248824fc..df7ec4461 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -515,7 +515,7 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
 
 class DummyBlockingFeedStorage(BlockingFeedStorage):
 
-    def __init__(self, uri):
+    def __init__(self, uri, *args, feed_options=None):
         self.path = file_uri_to_path(uri)
 
     def _store_in_thread(self, file):
@@ -541,7 +541,7 @@ class LogOnStoreFileStorage:
     It can be used to make sure `store` method is invoked.
     """
 
-    def __init__(self, uri):
+    def __init__(self, uri, feed_options=None):
         self.path = file_uri_to_path(uri)
         self.logger = getLogger()
 
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index f831ef5dc..04a594d03 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,6 +1,6 @@
 import unittest
 from unittest import mock
-from warnings import catch_warnings
+from warnings import catch_warnings, filterwarnings
 
 from w3lib.encoding import resolve_encoding
 
@@ -134,7 +134,9 @@ class BaseResponseTest(unittest.TestCase):
         assert isinstance(response.text, str)
         self._assert_response_encoding(response, encoding)
         self.assertEqual(response.body, body_bytes)
-        self.assertEqual(response.body_as_unicode(), body_unicode)
+        with catch_warnings():
+            filterwarnings("ignore", category=ScrapyDeprecationWarning)
+            self.assertEqual(response.body_as_unicode(), body_unicode)
         self.assertEqual(response.text, body_unicode)
 
     def _assert_response_encoding(self, response, encoding):
@@ -345,8 +347,10 @@ class TextResponseTest(BaseResponseTest):
         r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
 
         # check body_as_unicode
-        self.assertTrue(isinstance(r1.body_as_unicode(), str))
-        self.assertEqual(r1.body_as_unicode(), unicode_string)
+        with catch_warnings():
+            filterwarnings("ignore", category=ScrapyDeprecationWarning)
+            self.assertTrue(isinstance(r1.body_as_unicode(), str))
+            self.assertEqual(r1.body_as_unicode(), unicode_string)
 
         # check response.text
         self.assertTrue(isinstance(r1.text, str))
diff --git a/tests/test_item.py b/tests/test_item.py
index 78d204e34..c94bb44af 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,6 +1,6 @@
 import unittest
 from unittest import mock
-from warnings import catch_warnings
+from warnings import catch_warnings, filterwarnings
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta
@@ -328,16 +328,18 @@ class BaseItemTest(unittest.TestCase):
         class SubclassedItem(Item):
             pass
 
-        self.assertTrue(isinstance(BaseItem(), BaseItem))
-        self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))
-        self.assertTrue(isinstance(Item(), BaseItem))
-        self.assertTrue(isinstance(SubclassedItem(), BaseItem))
+        with catch_warnings():
+            filterwarnings("ignore", category=ScrapyDeprecationWarning)
+            self.assertTrue(isinstance(BaseItem(), BaseItem))
+            self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))
+            self.assertTrue(isinstance(Item(), BaseItem))
+            self.assertTrue(isinstance(SubclassedItem(), BaseItem))
 
-        # make sure internal checks using private _BaseItem class succeed
-        self.assertTrue(isinstance(BaseItem(), _BaseItem))
-        self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))
-        self.assertTrue(isinstance(Item(), _BaseItem))
-        self.assertTrue(isinstance(SubclassedItem(), _BaseItem))
+            # make sure internal checks using private _BaseItem class succeed
+            self.assertTrue(isinstance(BaseItem(), _BaseItem))
+            self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))
+            self.assertTrue(isinstance(Item(), _BaseItem))
+            self.assertTrue(isinstance(SubclassedItem(), _BaseItem))
 
     def test_deprecation_warning(self):
         """
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 35d35b45d..e47afa266 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -108,7 +108,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
 
         # ignore subclassing warnings
         with warnings.catch_warnings():
-            warnings.simplefilter('ignore', ScrapyDeprecationWarning)
+            warnings.simplefilter('ignore', MyWarning)
 
             class UserClass(Deprecated):
                 pass
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3115cc92f..4b3964154 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -5,8 +5,9 @@ import platform
 import unittest
 from datetime import datetime
 from itertools import count
-from warnings import catch_warnings
+from warnings import catch_warnings, filterwarnings
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
     WeakKeyCache, get_func_args, to_bytes, to_unicode,
@@ -160,7 +161,11 @@ class UtilsPythonTestCase(unittest.TestCase):
             pass
 
         _values = count()
-        wk = WeakKeyCache(lambda k: next(_values))
+
+        with catch_warnings():
+            filterwarnings("ignore", category=ScrapyDeprecationWarning)
+            wk = WeakKeyCache(lambda k: next(_values))
+
         k = _Weakme()
         v = wk[k]
         self.assertEqual(v, wk[k])

From 216dd37953112e360348e19eaf8cae45a52fe87a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 1 Jun 2021 11:16:40 -0300
Subject: [PATCH 387/716] Type hints for FormRequest

---
 scrapy/http/request/form.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index ef2eb3ba6..4465f40ae 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,6 +5,7 @@ This module implements the FormRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
+from typing import Optional, Type, TypeVar
 from urllib.parse import urljoin, urlencode
 
 import lxml.html
@@ -12,15 +13,18 @@ from parsel.selector import create_root_node
 from w3lib.html import strip_html5_whitespace
 
 from scrapy.http.request import Request
+from scrapy.http.response.text import TextResponse
 from scrapy.utils.python import to_bytes, is_listlike
 from scrapy.utils.response import get_base_url
 
 
+FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
+
+
 class FormRequest(Request):
     valid_form_methods = ['GET', 'POST']
 
-    def __init__(self, *args, **kwargs):
-        formdata = kwargs.pop('formdata', None)
+    def __init__(self, *args, formdata: Optional[dict] = None, **kwargs) -> None:
         if formdata and kwargs.get('method') is None:
             kwargs['method'] = 'POST'
 
@@ -36,9 +40,19 @@ class FormRequest(Request):
                 self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr)
 
     @classmethod
-    def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,
-                      clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs):
-
+    def from_response(
+        cls: Type[FormRequestTypeVar],
+        response: TextResponse,
+        formname: Optional[str] = None,
+        formid: Optional[str] = None,
+        formnumber: Optional[int] = 0,
+        formdata: Optional[dict] = None,
+        clickdata: Optional[dict] = None,
+        dont_click: bool = False,
+        formxpath: Optional[str] = None,
+        formcss: Optional[str] = None,
+        **kwargs,
+    ) -> FormRequestTypeVar:
         kwargs.setdefault('encoding', response.encoding)
 
         if formcss is not None:

From c594017e518af31edc24619bfc590907ba6280c7 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 1 Jun 2021 11:27:21 -0300
Subject: [PATCH 388/716] Type hints for private functions used by FormRequest

---
 scrapy/http/request/form.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 4465f40ae..781e3495a 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
 from typing import Optional, Type, TypeVar
 from urllib.parse import urljoin, urlencode
 
-import lxml.html
+from lxml.html import HTMLParser, FormElement
 from parsel.selector import create_root_node
 from w3lib.html import strip_html5_whitespace
 
@@ -72,7 +72,7 @@ class FormRequest(Request):
         return cls(url=url, method=method, formdata=formdata, **kwargs)
 
 
-def _get_form_url(form, url):
+def _get_form_url(form: FormElement, url: Optional[str]) -> str:
     if url is None:
         action = form.get('action')
         if action is None:
@@ -88,10 +88,15 @@ def _urlencode(seq, enc):
     return urlencode(values, doseq=True)
 
 
-def _get_form(response, formname, formid, formnumber, formxpath):
-    """Find the form element """
-    root = create_root_node(response.text, lxml.html.HTMLParser,
-                            base_url=get_base_url(response))
+def _get_form(
+    response: TextResponse,
+    formname: Optional[str],
+    formid: Optional[str],
+    formnumber: Optional[int],
+    formxpath: Optional[str],
+) -> FormElement:
+    """Find the wanted form element within the given response."""
+    root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response))
     forms = root.xpath('//form')
     if not forms:
         raise ValueError(f"No <form> element found in {response}")
@@ -119,8 +124,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
                     break
         raise ValueError(f'No <form> element found with {formxpath}')
 
-    # If we get here, it means that either formname was None
-    # or invalid
+    # If we get here, it means that either formname was None or invalid
     if formnumber is not None:
         try:
             form = forms[formnumber]

From 85f88a5710e51a3137ded72f5fdb1c3465480355 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 1 Jun 2021 12:02:16 -0300
Subject: [PATCH 389/716] More type hints for private functions used by
 FormRequest

---
 scrapy/http/request/form.py | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 781e3495a..1ad878c03 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,7 +5,7 @@ This module implements the FormRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
-from typing import Optional, Type, TypeVar
+from typing import List, Optional, Tuple, Type, TypeVar, Union
 from urllib.parse import urljoin, urlencode
 
 from lxml.html import HTMLParser, FormElement
@@ -20,11 +20,13 @@ from scrapy.utils.response import get_base_url
 
 FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
 
+FormdataType = Optional[Union[dict, List[Tuple[str, str]]]]
+
 
 class FormRequest(Request):
     valid_form_methods = ['GET', 'POST']
 
-    def __init__(self, *args, formdata: Optional[dict] = None, **kwargs) -> None:
+    def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None:
         if formdata and kwargs.get('method') is None:
             kwargs['method'] = 'POST'
 
@@ -46,7 +48,7 @@ class FormRequest(Request):
         formname: Optional[str] = None,
         formid: Optional[str] = None,
         formnumber: Optional[int] = 0,
-        formdata: Optional[dict] = None,
+        formdata: FormdataType = None,
         clickdata: Optional[dict] = None,
         dont_click: bool = False,
         formxpath: Optional[str] = None,
@@ -60,7 +62,7 @@ class FormRequest(Request):
             formxpath = HTMLTranslator().css_to_xpath(formcss)
 
         form = _get_form(response, formname, formid, formnumber, formxpath)
-        formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
+        formdata = _get_inputs(form, formdata, dont_click, clickdata)
         url = _get_form_url(form, kwargs.pop('url', None))
 
         method = kwargs.pop('method', form.method)
@@ -134,22 +136,27 @@ def _get_form(
             return form
 
 
-def _get_inputs(form, formdata, dont_click, clickdata, response):
+def _get_inputs(
+    form: FormElement,
+    formdata: FormdataType,
+    dont_click: bool,
+    clickdata: Optional[dict],
+) -> List[Tuple[str, str]]:
+    """Return a list of key-value pairs for the inputs found in the given form."""
     try:
         formdata_keys = dict(formdata or ()).keys()
     except (ValueError, TypeError):
         raise ValueError('formdata should be a dict or iterable of tuples')
 
     if not formdata:
-        formdata = ()
+        formdata = []
     inputs = form.xpath('descendant::textarea'
                         '|descendant::select'
                         '|descendant::input[not(@type) or @type['
                         ' not(re:test(., "^(?:submit|image|reset)$", "i"))'
                         ' and (../@checked or'
                         '  not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
-                        namespaces={
-                            "re": "http://exslt.org/regular-expressions"})
+                        namespaces={"re": "http://exslt.org/regular-expressions"})
     values = [(k, '' if v is None else v)
               for k, v in (_value(e) for e in inputs)
               if k and k not in formdata_keys]
@@ -160,7 +167,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
             values.append(clickable)
 
     if isinstance(formdata, dict):
-        formdata = formdata.items()
+        formdata = formdata.items()  # type: ignore[assignment]
 
     values.extend((k, v) for k, v in formdata if v is not None)
     return values
@@ -189,7 +196,7 @@ def _select_value(ele, n, v):
     return n, v
 
 
-def _get_clickable(clickdata, form):
+def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tuple[str, str]]:
     """
     Returns the clickable element specified in clickdata,
     if the latter is given. If not, it returns the first
@@ -201,7 +208,7 @@ def _get_clickable(clickdata, form):
         namespaces={"re": "http://exslt.org/regular-expressions"}
     ))
     if not clickables:
-        return
+        return None
 
     # If we don't have clickdata, we just use the first clickable element
     if clickdata is None:

From c9fecca010a3ddddd1145f65a78dde30b5c71a72 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 1 Jun 2021 12:25:26 -0300
Subject: [PATCH 390/716] More type hints

---
 scrapy/http/request/form.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 1ad878c03..c3e112041 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,10 +5,10 @@ This module implements the FormRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
-from typing import List, Optional, Tuple, Type, TypeVar, Union
+from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union
 from urllib.parse import urljoin, urlencode
 
-from lxml.html import HTMLParser, FormElement
+from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement
 from parsel.selector import create_root_node
 from w3lib.html import strip_html5_whitespace
 
@@ -83,7 +83,7 @@ def _get_form_url(form: FormElement, url: Optional[str]) -> str:
     return urljoin(form.base_url, url)
 
 
-def _urlencode(seq, enc):
+def _urlencode(seq: Iterable, enc: str) -> str:
     values = [(to_bytes(k, enc), to_bytes(v, enc))
               for k, vs in seq
               for v in (vs if is_listlike(vs) else [vs])]
@@ -157,9 +157,11 @@ def _get_inputs(
                         ' and (../@checked or'
                         '  not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
                         namespaces={"re": "http://exslt.org/regular-expressions"})
-    values = [(k, '' if v is None else v)
-              for k, v in (_value(e) for e in inputs)
-              if k and k not in formdata_keys]
+    values = [
+        (k, '' if v is None else v)
+        for k, v in (_value(e) for e in inputs)
+        if k and k not in formdata_keys
+    ]
 
     if not dont_click:
         clickable = _get_clickable(clickdata, form)
@@ -173,7 +175,7 @@ def _get_inputs(
     return values
 
 
-def _value(ele):
+def _value(ele: HtmlElement):
     n = ele.name
     v = ele.value
     if ele.tag == 'select':
@@ -181,7 +183,7 @@ def _value(ele):
     return n, v
 
 
-def _select_value(ele, n, v):
+def _select_value(ele: SelectElement, n: str, v: str):
     multiple = ele.multiple
     if v is None and not multiple:
         # Match browser behaviour on simple select tag without options selected
@@ -192,7 +194,8 @@ def _select_value(ele, n, v):
         # This is a workround to bug in lxml fixed 2.3.1
         # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
         selected_options = ele.xpath('.//option[@selected]')
-        v = [(o.get('value') or o.text or '').strip() for o in selected_options]
+        values = [(o.get('value') or o.text or '').strip() for o in selected_options]
+        return n, values
     return n, v
 
 

From 479260dca012b3d03221f2e6322008448d0fb8b5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Tue, 1 Jun 2021 12:52:46 -0300
Subject: [PATCH 391/716] Type hints for Request subclasses

---
 scrapy/http/request/json_request.py | 17 +++++++----------
 scrapy/http/request/rpc.py          |  4 ++--
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 04e80d897..dba3c3a82 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -8,9 +8,9 @@ See documentation in docs/topics/request-response.rst
 import copy
 import json
 import warnings
-from typing import Tuple
+from typing import Optional, Tuple
 
-from scrapy.http.request import Request
+from scrapy.http.request import Request, RequestTypeVar
 from scrapy.utils.deprecate import create_deprecated_class
 
 
@@ -18,8 +18,8 @@ class JsonRequest(Request):
 
     attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
 
-    def __init__(self, *args, **kwargs):
-        dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {}))
+    def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None:
+        dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {}
         dumps_kwargs.setdefault('sort_keys', True)
         self._dumps_kwargs = dumps_kwargs
 
@@ -29,10 +29,8 @@ class JsonRequest(Request):
 
         if body_passed and data_passed:
             warnings.warn('Both body and data passed. data will be ignored')
-
         elif not body_passed and data_passed:
             kwargs['body'] = self._dumps(data)
-
             if 'method' not in kwargs:
                 kwargs['method'] = 'POST'
 
@@ -41,23 +39,22 @@ class JsonRequest(Request):
         self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
 
     @property
-    def dumps_kwargs(self):
+    def dumps_kwargs(self) -> dict:
         return self._dumps_kwargs
 
-    def replace(self, *args, **kwargs):
+    def replace(self, *args, **kwargs) -> RequestTypeVar:
         body_passed = kwargs.get('body', None) is not None
         data = kwargs.pop('data', None)
         data_passed = data is not None
 
         if body_passed and data_passed:
             warnings.warn('Both body and data passed. data will be ignored')
-
         elif not body_passed and data_passed:
             kwargs['body'] = self._dumps(data)
 
         return super().replace(*args, **kwargs)
 
-    def _dumps(self, data):
+    def _dumps(self, data: dict) -> str:
         """Convert to JSON """
         return json.dumps(data, **self._dumps_kwargs)
 
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index c70912e49..06d98cea5 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -5,6 +5,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 import xmlrpc.client as xmlrpclib
+from typing import Optional
 
 from scrapy.http.request import Request
 from scrapy.utils.python import get_func_args
@@ -15,8 +16,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
 
 class XmlRpcRequest(Request):
 
-    def __init__(self, *args, **kwargs):
-        encoding = kwargs.get('encoding', None)
+    def __init__(self, *args, encoding: Optional[str] = None, **kwargs):
         if 'body' not in kwargs and 'params' in kwargs:
             kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs)
             kwargs['body'] = xmlrpclib.dumps(**kw)

From ce6447731a91e32faf17564d21ddc3b88e582f50 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Mon, 7 Jun 2021 13:25:04 -0300
Subject: [PATCH 392/716] Replace return type

---
 scrapy/http/request/__init__.py     | 4 ++--
 scrapy/http/request/json_request.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 8f00c20b7..7672dec00 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -126,10 +126,10 @@ class Request(object_ref):
 
     __repr__ = __str__
 
-    def copy(self) -> RequestTypeVar:
+    def copy(self) -> "Request":
         return self.replace()
 
-    def replace(self, *args, **kwargs) -> RequestTypeVar:
+    def replace(self, *args, **kwargs) -> "Request":
         """Create a new Request with the same attributes except for those given new values"""
         for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index dba3c3a82..728a2a104 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -10,7 +10,7 @@ import json
 import warnings
 from typing import Optional, Tuple
 
-from scrapy.http.request import Request, RequestTypeVar
+from scrapy.http.request import Request
 from scrapy.utils.deprecate import create_deprecated_class
 
 
@@ -42,7 +42,7 @@ class JsonRequest(Request):
     def dumps_kwargs(self) -> dict:
         return self._dumps_kwargs
 
-    def replace(self, *args, **kwargs) -> RequestTypeVar:
+    def replace(self, *args, **kwargs) -> Request:
         body_passed = kwargs.get('body', None) is not None
         data = kwargs.pop('data', None)
         data_passed = data is not None

From e876d8e38780a88e4c1fc060e3ead779222be83a Mon Sep 17 00:00:00 2001
From: Veniamin Gvozdikov <g.veniamin@googlemail.com>
Date: Fri, 11 Jun 2021 09:22:04 +0300
Subject: [PATCH 393/716] Rename scrapy-crawlera to scrapy-zyte-smartproxy
 (#5074)

---
 scrapy/core/downloader/handlers/http11.py |  6 +++---
 scrapy/core/downloader/handlers/http2.py  | 12 ++++++++----
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 073f35891..50486d13c 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -283,9 +283,9 @@ class ScrapyAgent:
             if omitConnectTunnel:
                 warnings.warn(
                     "Using HTTPS proxies in the noconnect mode is deprecated. "
-                    "If you use Zyte Smart Proxy Manager (formerly Crawlera), "
-                    "it doesn't require this mode anymore, so you should "
-                    "update scrapy-crawlera to 1.3.0+ and remove '?noconnect' "
+                    "If you use Zyte Smart Proxy Manager, it doesn't require "
+                    "this mode anymore, so you should update scrapy-crawlera "
+                    "to scrapy-zyte-smartproxy and remove '?noconnect' "
                     "from the Zyte Smart Proxy Manager URL.",
                     ScrapyDeprecationWarning,
                 )
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index e97c31e90..7bb88a193 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -72,10 +72,14 @@ class ScrapyH2Agent:
             proxy_host = proxy_host.decode()
             omit_connect_tunnel = b'noconnect' in proxy_params
             if omit_connect_tunnel:
-                warnings.warn("Using HTTPS proxies in the noconnect mode is not supported by the "
-                              "downloader handler. If you use Crawlera, it doesn't require this "
-                              "mode anymore, so you should update scrapy-crawlera to 1.3.0+ "
-                              "and remove '?noconnect' from the Crawlera URL.")
+                warnings.warn(
+                    "Using HTTPS proxies in the noconnect mode is not "
+                    "supported by the downloader handler. If you use Zyte "
+                    "Smart Proxy Manager, it doesn't require this mode "
+                    "anymore, so you should update scrapy-crawlera to "
+                    "scrapy-zyte-smartproxy and remove '?noconnect' from the "
+                    "Zyte Smart Proxy Manager URL."
+                )
 
             if scheme == b'https' and not omit_connect_tunnel:
                 # ToDo

From 28858574d92fd2170575bea83fc6abac6b45b63a Mon Sep 17 00:00:00 2001
From: pdt1931 <56060869+pdt1931@users.noreply.github.com>
Date: Fri, 11 Jun 2021 00:49:41 -0700
Subject: [PATCH 394/716] Add FAQ to code of Conduct (#5177)

* added to CODE_OF_CONDUCT.md to include link to FAQ about the code of conduct

* added to CODE_OF_CONDUCT.md to include link to FAQ about the code of conduct
---
 CODE_OF_CONDUCT.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index 652460383..902cd523e 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -72,3 +72,6 @@ available at [http://contributor-covenant.org/version/1/4][version].
 
 [homepage]: http://contributor-covenant.org
 [version]: http://contributor-covenant.org/version/1/4/
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

From de69d967f90bc70bc07f734b93a4e7b73f2a4aa9 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 11 Jun 2021 16:12:25 +0500
Subject: [PATCH 395/716] Fix async spider examples.

---
 docs/topics/coroutines.rst | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 6a39dcb5e..67f1a4098 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -77,14 +77,16 @@ coroutines, functions that return Deferreds and functions that return
 :term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
 This means you can use many useful Python libraries providing such code::
 
-    class MySpider(Spider):
+    class MySpiderDeferred(Spider):
         # ...
-        async def parse_with_deferred(self, response):
+        async def parse(self, response):
             additional_response = await treq.get('https://additional.url')
             additional_data = await treq.content(additional_response)
             # ... use response and additional_data to yield items and requests
 
-        async def parse_with_asyncio(self, response):
+    class MySpiderAsyncio(Spider):
+        # ...
+        async def parse(self, response):
             async with aiohttp.ClientSession() as session:
                 async with session.get('https://additional.url') as additional_response:
                     additional_data = await additional_response.text()

From 66e200423943a921326334a37777720c706ee2b5 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Mon, 14 Jun 2021 18:58:35 +0500
Subject: [PATCH 396/716] Fix a flake8 problem

---
 tests/test_http_request.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index ea32c6711..b610087bd 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -381,7 +381,8 @@ class FormRequestTest(RequestTest):
 
     def test_formdata_overrides_querystring(self):
         data = (('a', 'one'), ('a', 'two'), ('b', '2'))
-        url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0]
+        url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment',
+                                 method='GET', formdata=data).url.split('#')[0]
         fs = _qs(self.request_class(url, method='GET', formdata=data))
         self.assertEqual(set(fs[b'a']), {b'one', b'two'})
         self.assertEqual(fs[b'b'], [b'2'])

From 5044549c550876cc31fb2e15aaa5013e8fc333c3 Mon Sep 17 00:00:00 2001
From: Ajay Mittur <ajay.cs18@bmsce.ac.in>
Date: Mon, 14 Jun 2021 14:58:19 +0000
Subject: [PATCH 397/716] Update proxyScheme assignment

---
 scrapy/core/downloader/handlers/http11.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 15de8cdbd..d2f9084fc 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -319,7 +319,7 @@ class ScrapyAgent:
                     pool=self._pool,
                 )
             else:
-                proxyScheme = b'http' if not proxyScheme else proxyScheme
+                proxyScheme = proxyScheme or b'http'
                 proxyHost = to_bytes(proxyHost, encoding='ascii')
                 proxyPort = to_bytes(str(proxyPort), encoding='ascii')
                 proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))

From 7d653288e3d9369e4c007d223327f919b99b7737 Mon Sep 17 00:00:00 2001
From: ajaymittur28 <ajay.cs18@bmsce.ac.in>
Date: Mon, 14 Jun 2021 21:39:18 +0530
Subject: [PATCH 398/716] Update unittest

---
 tests/test_downloader_handlers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 67224ed53..9c11820e5 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -768,7 +768,7 @@ class Http11ProxyTestCase(HttpProxyTestCase):
         def _test(response):
             self.assertEqual(response.status, 200)
             self.assertEqual(response.url, request.url)
-            self.assertEqual(response.body, b'http://example.com')
+            self.assertEqual(response.body, self.expected_http_proxy_request_body)
 
         http_proxy = self.getURL('').replace('http://', '')
         request = Request('http://example.com', meta={'proxy': http_proxy})

From 812b4bb51855605a36bcfb166f66a121cc67f97c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= <ivan.prado@gmail.com>
Date: Wed, 23 Jun 2021 17:09:28 +0100
Subject: [PATCH 399/716] CloseSpider can be raised on spider_idle signal
 handler

---
 docs/topics/signals.rst |  3 +++
 scrapy/core/engine.py   | 16 +++++++++++-----
 scrapy/utils/signal.py  | 10 ++++------
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 3d838fb63..ee9d95836 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -268,6 +268,9 @@ spider_idle
     You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
     prevent the spider from being closed.
 
+    Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider`
+    exception to provide a custom spider closing reason.
+
     This signal does not support returning deferreds from its handlers.
 
     :param spider: the spider which has gone idle
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index dd3225082..949d89e76 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -15,7 +15,8 @@ from twisted.python.failure import Failure
 
 from scrapy import signals
 from scrapy.core.scraper import Scraper
-from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning
+from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning, \
+    CloseSpider
 from scrapy.http import Response, Request
 from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider
@@ -325,14 +326,19 @@ class ExecutionEngine:
         Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule.
         It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider
         exception, the spider is not closed until the next loop and this function is guaranteed to be called
-        (at least) once again.
+        (at least) once again. A handler can raise CloseSpider to provide a custom closing reason
         """
         assert self.spider is not None  # typing
-        res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider)
-        if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
+        expected_ex = (DontCloseSpider, CloseSpider)
+        res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex)
+        detected_ex = {ex: x.value
+                       for _, x in res for ex in expected_ex
+                       if isinstance(x, Failure) and isinstance(x.value, ex)}
+        if DontCloseSpider in detected_ex:
             return None
         if self.spider_is_idle():
-            self.close_spider(self.spider, reason='finished')
+            reason = detected_ex[CloseSpider].reason if CloseSpider in detected_ex else 'finished'
+            self.close_spider(self.spider, reason=reason)
 
     def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
         """Close (cancel) spider and clear all its outstanding requests"""
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index 115707182..62808f3ce 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -1,5 +1,5 @@
 """Helper functions for working with signals"""
-
+import collections
 import logging
 
 from twisted.internet.defer import DeferredList, Deferred
@@ -16,15 +16,13 @@ from scrapy.utils.log import failure_to_exc_info
 logger = logging.getLogger(__name__)
 
 
-class _IgnoredException(Exception):
-    pass
-
-
 def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     """Like pydispatcher.robust.sendRobust but it also logs errors and returns
     Failures instead of exceptions.
     """
-    dont_log = (named.pop('dont_log', _IgnoredException), StopDownload)
+    dont_log = named.pop('dont_log', ())
+    dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,)
+    dont_log += (StopDownload, )
     spider = named.get('spider', None)
     responses = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):

From ce445f20462463e49bd9dfd417aa7397d4c17f1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= <ivan.prado@gmail.com>
Date: Thu, 24 Jun 2021 09:56:05 +0100
Subject: [PATCH 400/716] Fix typing

---
 scrapy/core/engine.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 949d89e76..81a7a5068 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -337,8 +337,9 @@ class ExecutionEngine:
         if DontCloseSpider in detected_ex:
             return None
         if self.spider_is_idle():
-            reason = detected_ex[CloseSpider].reason if CloseSpider in detected_ex else 'finished'
-            self.close_spider(self.spider, reason=reason)
+            ex = detected_ex.get(CloseSpider, CloseSpider(reason='finished'))
+            assert isinstance(ex, CloseSpider)  # typing
+            self.close_spider(self.spider, reason=ex.reason)
 
     def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
         """Close (cancel) spider and clear all its outstanding requests"""

From 73ff9ffd64742bb4d64db376851b52cfc3658fcb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sat, 26 Jun 2021 08:58:29 +0200
Subject: [PATCH 401/716] spiders.rst: indent warnings into class descriptions

---
 docs/topics/spiders.rst | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index a3e9f410f..903fbd383 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -422,10 +422,9 @@ Crawling rules
    It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
    instance as first parameter.
 
-
-.. warning:: Because of its internal implementation, you must explicitly set
-   callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
-   unexpected behaviour can occur otherwise.
+   .. warning:: Because of its internal implementation, you must explicitly set
+      callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
+      unexpected behaviour can occur otherwise.
 
    .. versionadded:: 2.0
       The *errback* parameter.
@@ -557,10 +556,9 @@ XMLFeedSpider
         item IDs. It receives a list of results and the response which originated
         those results. It must return a list of results (items or requests).
 
-
-.. warning:: Because of its internal implementation, you must explicitly set
-   callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
-   unexpected behaviour can occur otherwise.
+    .. warning:: Because of its internal implementation, you must explicitly set
+       callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
+       unexpected behaviour can occur otherwise.
 
 
 XMLFeedSpider example

From f35970778b93033ee36379e0b8b55035aacaf918 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= <ivan.prado@gmail.com>
Date: Tue, 29 Jun 2021 13:21:38 +0100
Subject: [PATCH 402/716] Test case for raising CloseSpider on spider idle
 signal handler

---
 tests/test_engine.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index c200ded90..dc24f50fa 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -26,7 +26,7 @@ from twisted.web import server, static, util
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import ScrapyDeprecationWarning, CloseSpider
 from scrapy.http import Request
 from scrapy.item import Item, Field
 from scrapy.linkextractors import LinkExtractor
@@ -113,6 +113,18 @@ class ItemZeroDivisionErrorSpider(TestSpider):
     }
 
 
+class ChangeCloseReasonSpider(TestSpider):
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = cls(*args, **kwargs)
+        spider._set_crawler(crawler)
+        crawler.signals.connect(spider.spider_idle, signals.spider_idle)
+        return spider
+
+    def spider_idle(self):
+        raise CloseSpider(reason="custom_reason")
+
+
 def start_test_site(debug=False):
     root_dir = os.path.join(tests_datadir, "test_site")
     r = static.File(root_dir)
@@ -251,6 +263,13 @@ class EngineTest(unittest.TestCase):
         yield self.run.run()
         self._assert_items_error()
 
+    @defer.inlineCallbacks
+    def test_crawler_change_close_reason_on_idle(self):
+        self.run = CrawlerRun(ChangeCloseReasonSpider)
+        yield self.run.run()
+        self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'},
+                         self.run.signals_caught[signals.spider_closed])
+
     def _assert_visited_urls(self):
         must_be_visited = ["/", "/redirect", "/redirected",
                            "/item1.html", "/item2.html", "/item999.html"]

From e94d3ac173d2fcd269bdd8f06b6b1633953a225e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= <ivan.prado@gmail.com>
Date: Tue, 29 Jun 2021 13:40:43 +0100
Subject: [PATCH 403/716] Expanded doc for idle signal

---
 docs/topics/signals.rst | 6 +++++-
 scrapy/core/engine.py   | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index ee9d95836..530af1e37 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -269,7 +269,11 @@ spider_idle
     prevent the spider from being closed.
 
     Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider`
-    exception to provide a custom spider closing reason.
+    exception to provide a custom spider closing reason. An
+    idle handler is the perfect place to put some code that assesses
+    the final spider results and update the final closing reason
+    accordingly (e.g. setting it to 'too_few_results' instead of
+    'finished').
 
     This signal does not support returning deferreds from its handlers.
 
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 81a7a5068..0b34d213d 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -326,7 +326,7 @@ class ExecutionEngine:
         Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule.
         It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider
         exception, the spider is not closed until the next loop and this function is guaranteed to be called
-        (at least) once again. A handler can raise CloseSpider to provide a custom closing reason
+        (at least) once again. A handler can raise CloseSpider to provide a custom closing reason.
         """
         assert self.spider is not None  # typing
         expected_ex = (DontCloseSpider, CloseSpider)

From 7597d860c86e0ddfaf8e3ce57492106c907245f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?=
 <ivanprado@users.noreply.github.com>
Date: Thu, 8 Jul 2021 12:39:17 +0100
Subject: [PATCH 404/716] Update scrapy/core/engine.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/core/engine.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 0b34d213d..b7f91d744 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -15,8 +15,11 @@ from twisted.python.failure import Failure
 
 from scrapy import signals
 from scrapy.core.scraper import Scraper
-from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning, \
-    CloseSpider
+from scrapy.exceptions import (
+    CloseSpider,
+    DontCloseSpider, 
+    ScrapyDeprecationWarning,
+)   
 from scrapy.http import Response, Request
 from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider

From 6b8f694653b0eec95055d50fafde0ef2e0ec8bc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?=
 <ivanprado@users.noreply.github.com>
Date: Thu, 8 Jul 2021 12:40:02 +0100
Subject: [PATCH 405/716] Update scrapy/core/engine.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/core/engine.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index b7f91d744..b04fedce6 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -334,9 +334,12 @@ class ExecutionEngine:
         assert self.spider is not None  # typing
         expected_ex = (DontCloseSpider, CloseSpider)
         res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex)
-        detected_ex = {ex: x.value
-                       for _, x in res for ex in expected_ex
-                       if isinstance(x, Failure) and isinstance(x.value, ex)}
+        detected_ex = {
+            ex: x.value
+            for _, x in res
+            for ex in expected_ex
+            if isinstance(x, Failure) and isinstance(x.value, ex)
+        }
         if DontCloseSpider in detected_ex:
             return None
         if self.spider_is_idle():

From eca641aa3da335781c373e3d50deb78837044c16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?=
 <ivanprado@users.noreply.github.com>
Date: Thu, 8 Jul 2021 12:40:20 +0100
Subject: [PATCH 406/716] Update tests/test_engine.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 tests/test_engine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index dc24f50fa..92bf45f25 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -26,7 +26,7 @@ from twisted.web import server, static, util
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine
-from scrapy.exceptions import ScrapyDeprecationWarning, CloseSpider
+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.item import Item, Field
 from scrapy.linkextractors import LinkExtractor

From cb08e3644b081febecd90eec9391367a2d13ee5b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Thu, 8 Jul 2021 09:22:21 -0300
Subject: [PATCH 407/716] Remove trailing whitespaces

---
 scrapy/core/engine.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index b04fedce6..f9de7ee23 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -17,9 +17,9 @@ from scrapy import signals
 from scrapy.core.scraper import Scraper
 from scrapy.exceptions import (
     CloseSpider,
-    DontCloseSpider, 
+    DontCloseSpider,
     ScrapyDeprecationWarning,
-)   
+)
 from scrapy.http import Response, Request
 from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider

From c062ed017a89e40c41140ff2782043776b26165b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Mon, 12 Jul 2021 13:34:22 -0300
Subject: [PATCH 408/716] [CI] fail-fast: false (#5200)

---
 .github/workflows/checks.yml        | 1 +
 .github/workflows/tests-macos.yml   | 1 +
 .github/workflows/tests-ubuntu.yml  | 1 +
 .github/workflows/tests-windows.yml | 1 +
 4 files changed, 4 insertions(+)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 02c647da9..e7080db9a 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -5,6 +5,7 @@ jobs:
   checks:
     runs-on: ubuntu-18.04
     strategy:
+      fail-fast: false
       matrix:
         include:
         - python-version: 3.9
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 4f8f7a19d..095ca1013 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -5,6 +5,7 @@ jobs:
   tests:
     runs-on: macos-10.15
     strategy:
+      fail-fast: false
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
 
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index df5ee9d69..b42e8b127 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -5,6 +5,7 @@ jobs:
   tests:
     runs-on: ubuntu-18.04
     strategy:
+      fail-fast: false
       matrix:
         include:
         - python-version: 3.7
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 5459a845b..30fda33e8 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -5,6 +5,7 @@ jobs:
   tests:
     runs-on: windows-latest
     strategy:
+      fail-fast: false
       matrix:
         include:
         - python-version: 3.6

From 4ddc9d6b55fa708becfd70812ea44eb0c6837638 Mon Sep 17 00:00:00 2001
From: D R Siddhartha <siddharthadr11@gmail.com>
Date: Tue, 13 Jul 2021 20:52:29 +0530
Subject: [PATCH 409/716] Feeds: Item Filters (#5178)

---
 docs/topics/feed-exports.rst    |  54 +++++++++++++++
 scrapy/extensions/feedexport.py |  48 ++++++++++++-
 tests/test_feedexport.py        | 118 ++++++++++++++++++++++++++++++--
 3 files changed, 214 insertions(+), 6 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 26c247cdd..7a4b054e9 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -268,6 +268,45 @@ in multiple files, with the specified maximum item count per file. That way, as
 soon as a file reaches the maximum item count, that file is delivered to the
 feed URI, allowing item delivery to start way before the end of the crawl.
 
+.. _item-filter:
+
+Item filtering
+==============
+
+.. versionadded:: VERSION
+
+You can filter items that you want to allow for a particular feed by using the
+``item_classes`` option in :ref:`feeds options <feed-options>`. Only items of
+the specified types will be added to the feed.
+
+The ``item_classes`` option is implemented by the :class:`~scrapy.extensions.feedexport.ItemFilter`
+class, which is the default value of the ``item_filter`` :ref:`feed option <feed-options>`.
+
+You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s
+method ``accepts`` and taking ``feed_options`` as an argument.
+
+For instance::
+
+    class MyCustomFilter:
+
+        def __init__(self, feed_options):
+            self.feed_options = feed_options
+
+        def accepts(self, item):
+            if "field1" in item and item["field1"] == "expected_data":
+                return True
+            return False
+
+
+You can assign your custom filtering class to the ``item_filter`` :ref:`option of a feed <feed-options>`.
+See :setting:`FEEDS` for examples.
+
+ItemFilter
+----------
+
+.. autoclass:: scrapy.extensions.feedexport.ItemFilter
+   :members:
+
 
 Settings
 ========
@@ -311,6 +350,7 @@ For instance::
             'format': 'json',
             'encoding': 'utf8',
             'store_empty': False,
+            'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'],
             'fields': None,
             'indent': 4,
             'item_export_kwargs': {
@@ -320,12 +360,14 @@ For instance::
         '/home/user/documents/items.xml': {
             'format': 'xml',
             'fields': ['name', 'price'],
+            'item_filter': MyCustomFilter1,
             'encoding': 'latin1',
             'indent': 8,
         },
         pathlib.Path('items.csv'): {
             'format': 'csv',
             'fields': ['price', 'name'],
+            'item_filter': 'myproject.filters.MyCustomFilter2',
         },
     }
 
@@ -347,6 +389,18 @@ as a fallback value if that key is not provided for a specific feed definition:
 
 -   ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
 
+-   ``item_classes``: list of :ref:`item classes <topics-items>` to export.
+
+    If undefined or empty, all items are exported.
+
+    .. versionadded:: VERSION
+
+-   ``item_filter``: a :ref:`filter class <item-filter>` to filter items to export.
+
+    :class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.
+
+    .. versionadded:: VERSION
+
 -   ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
 
 -   ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index bec114707..89dca12f4 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -46,6 +46,38 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
     return builder(*preargs, uri, *args, **kwargs)
 
 
+class ItemFilter:
+    """
+    This will be used by FeedExporter to decide if an item should be allowed
+    to be exported to a particular feed.
+
+    :param feed_options: feed specific options passed from FeedExporter
+    :type feed_options: dict
+    """
+
+    def __init__(self, feed_options):
+        self.feed_options = feed_options
+        self.item_classes = set()
+
+        if 'item_classes' in self.feed_options:
+            for item_class in self.feed_options['item_classes']:
+                self.item_classes.add(load_object(item_class))
+
+    def accepts(self, item):
+        """
+        Return ``True`` if `item` should be exported or ``False`` otherwise.
+
+        :param item: scraped item which user wants to check if is acceptable
+        :type item: :ref:`Scrapy items <topics-items>`
+        :return: `True` if accepted, `False` otherwise
+        :rtype: bool
+        """
+        if self.item_classes:
+            return isinstance(item, tuple(self.item_classes))
+
+        return True    # accept all items if none declared in item_classes
+
+
 class IFeedStorage(Interface):
     """Interface that all Feed Storages must implement"""
 
@@ -215,7 +247,7 @@ class FTPFeedStorage(BlockingFeedStorage):
 
 
 class _FeedSlot:
-    def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
+    def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template, filter):
         self.file = file
         self.exporter = exporter
         self.storage = storage
@@ -225,6 +257,7 @@ class _FeedSlot:
         self.store_empty = store_empty
         self.uri_template = uri_template
         self.uri = uri
+        self.filter = filter
         # flags
         self.itemcount = 0
         self._exporting = False
@@ -255,6 +288,7 @@ class FeedExporter:
         self.settings = crawler.settings
         self.feeds = {}
         self.slots = []
+        self.filters = {}
 
         if not self.settings['FEEDS'] and not self.settings['FEED_URI']:
             raise NotConfigured
@@ -269,12 +303,14 @@ class FeedExporter:
             uri = str(self.settings['FEED_URI'])  # handle pathlib.Path objects
             feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
             self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
+            self.filters[uri] = self._load_filter(feed_options)
         # End: Backward compatibility for FEED_URI and FEED_FORMAT settings
 
         # 'FEEDS' setting takes precedence over 'FEED_URI'
         for uri, feed_options in self.settings.getdict('FEEDS').items():
             uri = str(uri)  # handle pathlib.Path objects
             self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
+            self.filters[uri] = self._load_filter(feed_options)
 
         self.storages = self._load_components('FEED_STORAGES')
         self.exporters = self._load_components('FEED_EXPORTERS')
@@ -368,6 +404,7 @@ class FeedExporter:
             store_empty=feed_options['store_empty'],
             batch_id=batch_id,
             uri_template=uri_template,
+            filter=self.filters[uri_template]
         )
         if slot.store_empty:
             slot.start_exporting()
@@ -376,6 +413,10 @@ class FeedExporter:
     def item_scraped(self, item, spider):
         slots = []
         for slot in self.slots:
+            if not slot.filter.accepts(item):
+                slots.append(slot)    # if slot doesn't accept item, continue with next slot
+                continue
+
             slot.start_exporting()
             slot.exporter.export_item(item)
             slot.itemcount += 1
@@ -486,3 +527,8 @@ class FeedExporter:
         uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
         uripar_function(params, spider)
         return params
+
+    def _load_filter(self, feed_options):
+        # load the item filter if declared else load the default filter class
+        item_filter_class = load_object(feed_options.get("item_filter", ItemFilter))
+        return item_filter_class(feed_options)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index df7ec4461..81437b011 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -561,6 +561,10 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         egg = scrapy.Field()
         baz = scrapy.Field()
 
+    class MyItem2(scrapy.Item):
+        foo = scrapy.Field()
+        hello = scrapy.Field()
+
     def _random_temp_filename(self, inter_dir=''):
         chars = [random.choice(ascii_letters + digits) for _ in range(15)]
         filename = ''.join(chars)
@@ -888,13 +892,9 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_export_multiple_item_classes(self):
 
-        class MyItem2(scrapy.Item):
-            foo = scrapy.Field()
-            hello = scrapy.Field()
-
         items = [
             self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
-            MyItem2({'hello': 'world2', 'foo': 'bar2'}),
+            self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
             self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}),
             {'hello': 'world4', 'egg': 'spam4'},
         ]
@@ -929,6 +929,114 @@ class FeedExportTest(FeedExportTestBase):
         yield self.assertExported(items, header, rows,
                                   settings=settings, ordered=True)
 
+    @defer.inlineCallbacks
+    def test_export_based_on_item_classes(self):
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
+            {'hello': 'world3', 'egg': 'spam3'},
+        ]
+
+        formats = {
+            'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n',
+            'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]',
+            'jsonlines': (
+                b'{"foo": "bar1", "egg": "spam1"}\n'
+                b'{"hello": "world2", "foo": "bar2"}\n'
+            ),
+            'xml': (
+                b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
+                b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'
+                b'world2</hello><foo>bar2</foo></item>\n<item><hello>world3'
+                b'</hello><egg>spam3</egg></item>\n</items>'
+            ),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {
+                    'format': 'csv',
+                    'item_classes': [self.MyItem],
+                },
+                self._random_temp_filename(): {
+                    'format': 'json',
+                    'item_classes': [self.MyItem2],
+                },
+                self._random_temp_filename(): {
+                    'format': 'jsonlines',
+                    'item_classes': [self.MyItem, self.MyItem2],
+                },
+                self._random_temp_filename(): {
+                    'format': 'xml',
+                },
+            },
+        }
+
+        data = yield self.exported_data(items, settings)
+        for fmt, expected in formats.items():
+            self.assertEqual(expected, data[fmt])
+
+    @defer.inlineCallbacks
+    def test_export_based_on_custom_filters(self):
+        items = [
+            self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+            self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
+            {'hello': 'world3', 'egg': 'spam3'},
+        ]
+
+        MyItem = self.MyItem
+
+        class CustomFilter1:
+            def __init__(self, feed_options):
+                pass
+
+            def accepts(self, item):
+                return isinstance(item, MyItem)
+
+        class CustomFilter2(scrapy.extensions.feedexport.ItemFilter):
+            def accepts(self, item):
+                if 'foo' not in item.fields:
+                    return False
+                return True
+
+        class CustomFilter3(scrapy.extensions.feedexport.ItemFilter):
+            def accepts(self, item):
+                if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1":
+                    return True
+                return False
+
+        formats = {
+            'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]',
+            'xml': (
+                b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
+                b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'
+                b'world2</hello><foo>bar2</foo></item>\n</items>'
+            ),
+            'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n',
+        }
+
+        settings = {
+            'FEEDS': {
+                self._random_temp_filename(): {
+                    'format': 'json',
+                    'item_filter': CustomFilter1,
+                },
+                self._random_temp_filename(): {
+                    'format': 'xml',
+                    'item_filter': CustomFilter2,
+                },
+                self._random_temp_filename(): {
+                    'format': 'jsonlines',
+                    'item_classes': [self.MyItem, self.MyItem2],
+                    'item_filter': CustomFilter3,
+                },
+            },
+        }
+
+        data = yield self.exported_data(items, settings)
+        for fmt, expected in formats.items():
+            self.assertEqual(expected, data[fmt])
+
     @defer.inlineCallbacks
     def test_export_dicts(self):
         # When dicts are used, only keys from the first row are used as

From fcc6becc586e6f895cdeed66a579d5735d215de1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?T=C3=BCrkalp=20Burak=20KAYRANCIO=C4=9ELU?=
 <bkayranci@users.noreply.github.com>
Date: Wed, 14 Jul 2021 11:00:43 +0300
Subject: [PATCH 410/716] S3FeedStorage: allow custom endpoint (#4998)

Co-authored-by: Andrey Rahmatullin <wrar@wrar.name>
---
 docs/topics/feed-exports.rst    |  3 ++-
 scrapy/extensions/feedexport.py |  7 ++++--
 tests/test_feedexport.py        | 44 +++++++++++++++++++++++++++++++--
 3 files changed, 49 insertions(+), 5 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 7a4b054e9..af7fce852 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -201,9 +201,10 @@ passed through the following settings:
 -   :setting:`AWS_ACCESS_KEY_ID`
 -   :setting:`AWS_SECRET_ACCESS_KEY`
 
-You can also define a custom ACL for exported feeds using this setting:
+You can also define a custom ACL and custom endpoint for exported feeds using this setting:
 
 -   :setting:`FEED_STORAGE_S3_ACL`
+-   :setting:`AWS_ENDPOINT_URL`
 
 This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 89dca12f4..84a79e32d 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -150,7 +150,7 @@ class FileFeedStorage:
 
 class S3FeedStorage(BlockingFeedStorage):
 
-    def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,
+    def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *,
                  feed_options=None):
         if not is_botocore_available():
             raise NotConfigured('missing botocore library')
@@ -160,11 +160,13 @@ class S3FeedStorage(BlockingFeedStorage):
         self.secret_key = u.password or secret_key
         self.keyname = u.path[1:]  # remove first "/"
         self.acl = acl
+        self.endpoint_url = endpoint_url
         import botocore.session
         session = botocore.session.get_session()
         self.s3_client = session.create_client(
             's3', aws_access_key_id=self.access_key,
-            aws_secret_access_key=self.secret_key)
+            aws_secret_access_key=self.secret_key,
+            endpoint_url=self.endpoint_url)
         if feed_options and feed_options.get('overwrite', True) is False:
             logger.warning('S3 does not support appending to files. To '
                            'suppress this warning, remove the overwrite '
@@ -178,6 +180,7 @@ class S3FeedStorage(BlockingFeedStorage):
             access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
             secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
             acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
+            endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None,
             feed_options=feed_options,
         )
 
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 81437b011..da0b2c786 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -326,6 +326,17 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, 'custom-acl')
 
+    def test_init_with_endpoint_url(self):
+        storage = S3FeedStorage(
+            's3://mybucket/export.csv',
+            'access_key',
+            'secret_key',
+            endpoint_url='https://example.com'
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.endpoint_url, 'https://example.com')
+
     def test_from_crawler_without_acl(self):
         settings = {
             'AWS_ACCESS_KEY_ID': 'access_key',
@@ -340,6 +351,20 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, None)
 
+    def test_without_endpoint_url(self):
+        settings = {
+            'AWS_ACCESS_KEY_ID': 'access_key',
+            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+        }
+        crawler = get_crawler(settings_dict=settings)
+        storage = S3FeedStorage.from_crawler(
+            crawler,
+            's3://mybucket/export.csv',
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.endpoint_url, None)
+
     def test_from_crawler_with_acl(self):
         settings = {
             'AWS_ACCESS_KEY_ID': 'access_key',
@@ -355,6 +380,21 @@ class S3FeedStorageTest(unittest.TestCase):
         self.assertEqual(storage.secret_key, 'secret_key')
         self.assertEqual(storage.acl, 'custom-acl')
 
+    def test_from_crawler_with_endpoint_url(self):
+        settings = {
+            'AWS_ACCESS_KEY_ID': 'access_key',
+            'AWS_SECRET_ACCESS_KEY': 'secret_key',
+            'AWS_ENDPOINT_URL': 'https://example.com',
+        }
+        crawler = get_crawler(settings_dict=settings)
+        storage = S3FeedStorage.from_crawler(
+            crawler,
+            's3://mybucket/export.csv'
+        )
+        self.assertEqual(storage.access_key, 'access_key')
+        self.assertEqual(storage.secret_key, 'secret_key')
+        self.assertEqual(storage.endpoint_url, 'https://example.com')
+
     @defer.inlineCallbacks
     def test_store_botocore_without_acl(self):
         skip_if_no_boto()
@@ -1917,8 +1957,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
 
 class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
 
-    def __init__(self, uri, access_key, secret_key, acl):
-        super().__init__(uri, access_key, secret_key, acl)
+    def __init__(self, uri, access_key, secret_key, acl, endpoint_url):
+        super().__init__(uri, access_key, secret_key, acl, endpoint_url)
 
 
 class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):

From d7deba7e89242774ae712d87eb7bb331759a731f Mon Sep 17 00:00:00 2001
From: Marlena Chatzigrigoriou
 <56519084+marlenachatzigrigoriou@users.noreply.github.com>
Date: Wed, 14 Jul 2021 11:34:28 +0300
Subject: [PATCH 411/716] Document all import paths and use the shortest in
 examples (#5099)

---
 docs/faq.rst                          |  4 +-
 docs/intro/tutorial.rst               | 32 +++++++--------
 docs/topics/api.rst                   |  4 +-
 docs/topics/contracts.rst             |  4 +-
 docs/topics/coroutines.rst            |  2 +-
 docs/topics/debug.rst                 |  2 +-
 docs/topics/developer-tools.rst       |  2 +-
 docs/topics/downloader-middleware.rst | 58 +++++++++++++--------------
 docs/topics/dynamic-content.rst       | 14 +++----
 docs/topics/exporters.rst             |  4 +-
 docs/topics/feed-exports.rst          |  4 +-
 docs/topics/item-pipeline.rst         |  6 +--
 docs/topics/items.rst                 | 10 +++--
 docs/topics/jobs.rst                  |  4 +-
 docs/topics/leaks.rst                 | 16 ++++----
 docs/topics/logging.rst               |  2 +-
 docs/topics/media-pipeline.rst        |  6 +--
 docs/topics/request-response.rst      | 16 ++++----
 docs/topics/selectors.rst             |  8 ++--
 docs/topics/settings.rst              | 16 ++++----
 docs/topics/shell.rst                 |  8 ++--
 docs/topics/signals.rst               | 54 ++++++++++++-------------
 docs/topics/spider-middleware.rst     | 26 ++++++------
 docs/topics/spiders.rst               | 43 ++++++++++----------
 24 files changed, 175 insertions(+), 170 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 9709885f6..16903daea 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -149,7 +149,7 @@ How can I prevent memory errors due to many allowed domains?
 ------------------------------------------------------------
 
 If you have a spider with a long list of
-:attr:`~scrapy.spiders.Spider.allowed_domains` (e.g. 50,000+), consider
+:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider
 replacing the default
 :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware
 with a :ref:`custom spider middleware <custom-spider-middleware>` that requires
@@ -157,7 +157,7 @@ less memory. For example:
 
 -   If your domain names are similar enough, use your own regular expression
     instead joining the strings in
-    :attr:`~scrapy.spiders.Spider.allowed_domains` into a complex regular
+    :attr:`~scrapy.Spider.allowed_domains` into a complex regular
     expression.
 
 -   If you can `meet the installation requirements`_, use pyre2_ instead of
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 740e47d0c..438f3d6df 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -78,7 +78,7 @@ Our first Spider
 
 Spiders are classes that you define and that Scrapy uses to scrape information
 from a website (or a group of websites). They must subclass
-:class:`~scrapy.spiders.Spider` and define the initial requests to make,
+:class:`~scrapy.Spider` and define the initial requests to make,
 optionally how to follow links in the pages, and how to parse the downloaded
 page content to extract data.
 
@@ -107,26 +107,26 @@ This is the code for our first Spider. Save it in a file named
             self.log(f'Saved file {filename}')
 
 
-As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.spiders.Spider>`
+As you can see, our Spider subclasses :class:`scrapy.Spider <scrapy.Spider>`
 and defines some attributes and methods:
 
-* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be
+* :attr:`~scrapy.Spider.name`: identifies the Spider. It must be
   unique within a project, that is, you can't set the same name for different
   Spiders.
 
-* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of
+* :meth:`~scrapy.Spider.start_requests`: must return an iterable of
   Requests (you can return a list of requests or write a generator function)
   which the Spider will begin to crawl from. Subsequent requests will be
   generated successively from these initial requests.
 
-* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle
+* :meth:`~scrapy.Spider.parse`: a method that will be called to handle
   the response downloaded for each of the requests made. The response parameter
   is an instance of :class:`~scrapy.http.TextResponse` that holds
   the page content and has further helpful methods to handle it.
 
-  The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting
+  The :meth:`~scrapy.Spider.parse` method usually parses the response, extracting
   the scraped data as dicts and also finding new URLs to
-  follow and creating new requests (:class:`~scrapy.http.Request`) from them.
+  follow and creating new requests (:class:`~scrapy.Request`) from them.
 
 How to run our spider
 ---------------------
@@ -162,7 +162,7 @@ for the respective URLs, as our ``parse`` method instructs.
 What just happened under the hood?
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Scrapy schedules the :class:`scrapy.Request <scrapy.http.Request>` objects
+Scrapy schedules the :class:`scrapy.Request <scrapy.Request>` objects
 returned by the ``start_requests`` method of the Spider. Upon receiving a
 response for each one, it instantiates :class:`~scrapy.http.Response` objects
 and calls the callback method associated with the request (in this case, the
@@ -171,11 +171,11 @@ and calls the callback method associated with the request (in this case, the
 
 A shortcut to the start_requests method
 ---------------------------------------
-Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method
-that generates :class:`scrapy.Request <scrapy.http.Request>` objects from URLs,
-you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute
+Instead of implementing a :meth:`~scrapy.Spider.start_requests` method
+that generates :class:`scrapy.Request <scrapy.Request>` objects from URLs,
+you can just define a :attr:`~scrapy.Spider.start_urls` class attribute
 with a list of URLs. This list will then be used by the default implementation
-of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests
+of :meth:`~scrapy.Spider.start_requests` to create the initial requests
 for your spider::
 
     import scrapy
@@ -194,9 +194,9 @@ for your spider::
             with open(filename, 'wb') as f:
                 f.write(response.body)
 
-The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each
+The :meth:`~scrapy.Spider.parse` method will be called to handle each
 of the requests for those URLs, even though we haven't explicitly told Scrapy
-to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's
+to do so. This happens because :meth:`~scrapy.Spider.parse` is Scrapy's
 default callback method, which is called for requests without an explicitly
 assigned callback.
 
@@ -248,7 +248,7 @@ object:
 
 The result of running ``response.css('title')`` is a list-like object called
 :class:`~scrapy.selector.SelectorList`, which represents a list of
-:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements
+:class:`~scrapy.Selector` objects that wrap around XML/HTML elements
 and allow you to run further queries to fine-grain the selection or extract the
 data.
 
@@ -670,7 +670,7 @@ the pagination links with the ``parse`` callback as we saw before.
 Here we're passing callbacks to
 :meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` as positional
 arguments to make the code shorter; it also works for
-:class:`~scrapy.http.Request`.
+:class:`~scrapy.Request`.
 
 The ``parse_author`` callback defines a helper function to extract and cleanup the
 data from a CSS query and yields the Python dict with the author data.
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 445b2979f..900b19c7a 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -29,7 +29,7 @@ how you :ref:`configure the downloader middlewares
 .. class:: Crawler(spidercls, settings)
 
     The Crawler object must be instantiated with a
-    :class:`scrapy.spiders.Spider` subclass and a
+    :class:`scrapy.Spider` subclass and a
     :class:`scrapy.settings.Settings` object.
 
     .. attribute:: settings
@@ -196,7 +196,7 @@ SpiderLoader API
        match the request's url against the domains of the spiders.
 
        :param request: queried request
-       :type request: :class:`~scrapy.http.Request` instance
+       :type request: :class:`~scrapy.Request` instance
 
 .. _topics-api-signals:
 
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index e61421bf1..ef296dc9e 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -37,7 +37,7 @@ This callback is tested using three built-in contracts:
 
 .. class:: CallbackKeywordArgumentsContract
 
-    This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.http.Request.cb_kwargs>`
+    This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`
     attribute for the sample request. It must be a valid JSON dictionary.
     ::
 
@@ -88,7 +88,7 @@ override three methods:
     .. method:: Contract.adjust_request_args(args)
 
         This receives a ``dict`` as an argument containing default arguments
-        for request object. :class:`~scrapy.http.Request` is used by default,
+        for request object. :class:`~scrapy.Request` is used by default,
         but this can be changed with the ``request_cls`` attribute.
         If multiple contracts in chain have this attribute defined, the last one is used.
 
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 3b1549bd3..0904637b0 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -15,7 +15,7 @@ Supported callables
 The following callables may be defined as coroutines using ``async def``, and
 hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
--   :class:`~scrapy.http.Request` callbacks.
+-   :class:`~scrapy.Request` callbacks.
 
     .. note:: The callback output is not processed until the whole callback
         finishes.
diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index d75f17301..4d452b4df 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -36,7 +36,7 @@ Consider the following Scrapy spider below::
 
 Basically this is a simple spider which parses two pages of items (the
 start_urls). Items also have a details page with additional information, so we
-use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a
+use the ``cb_kwargs`` functionality of :class:`~scrapy.Request` to pass a
 partially populated item.
 
 
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index c83b1a9d9..057b1ec62 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -274,7 +274,7 @@ In more complex websites, it could be difficult to easily reproduce the
 requests, as we could need to add ``headers`` or ``cookies`` to make it work.
 In those cases you can export the requests in `cURL <https://curl.haxx.se/>`_
 format, by right-clicking on each of them in the network tool and using the
-:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent
+:meth:`~scrapy.Request.from_curl()` method to generate an equivalent
 request::
 
     from scrapy import Request
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index b539c23df..80c6c2c37 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -76,7 +76,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
       middleware.
 
       :meth:`process_request` should either: return ``None``, return a
-      :class:`~scrapy.http.Response` object, return a :class:`~scrapy.http.Request`
+      :class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request`
       object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`.
 
       If it returns ``None``, Scrapy will continue processing this request, executing all
@@ -88,7 +88,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
       or the appropriate download function; it'll return that response. The :meth:`process_response`
       methods of installed middleware is always called on every response.
 
-      If it returns a :class:`~scrapy.http.Request` object, Scrapy will stop calling
+      If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling
       process_request methods and reschedule the returned request. Once the newly returned
       request is performed, the appropriate middleware chain will be called on
       the downloaded response.
@@ -100,22 +100,22 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
       ignored and not logged (unlike other exceptions).
 
       :param request: the request being processed
-      :type request: :class:`~scrapy.http.Request` object
+      :type request: :class:`~scrapy.Request` object
 
       :param spider: the spider for which this request is intended
-      :type spider: :class:`~scrapy.spiders.Spider` object
+      :type spider: :class:`~scrapy.Spider` object
 
    .. method:: process_response(request, response, spider)
 
       :meth:`process_response` should either: return a :class:`~scrapy.http.Response`
-      object, return a :class:`~scrapy.http.Request` object or
+      object, return a :class:`~scrapy.Request` object or
       raise a :exc:`~scrapy.exceptions.IgnoreRequest` exception.
 
       If it returns a :class:`~scrapy.http.Response` (it could be the same given
       response, or a brand-new one), that response will continue to be processed
       with the :meth:`process_response` of the next middleware in the chain.
 
-      If it returns a :class:`~scrapy.http.Request` object, the middleware chain is
+      If it returns a :class:`~scrapy.Request` object, the middleware chain is
       halted and the returned request is rescheduled to be downloaded in the future.
       This is the same behavior as if a request is returned from :meth:`process_request`.
 
@@ -124,13 +124,13 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
       exception, it is ignored and not logged (unlike other exceptions).
 
       :param request: the request that originated the response
-      :type request: is a :class:`~scrapy.http.Request` object
+      :type request: is a :class:`~scrapy.Request` object
 
       :param response: the response being processed
       :type response: :class:`~scrapy.http.Response` object
 
       :param spider: the spider for which this response is intended
-      :type spider: :class:`~scrapy.spiders.Spider` object
+      :type spider: :class:`~scrapy.Spider` object
 
    .. method:: process_exception(request, exception, spider)
 
@@ -139,7 +139,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
       exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception)
 
       :meth:`process_exception` should return: either ``None``,
-      a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.http.Request` object.
+      a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object.
 
       If it returns ``None``, Scrapy will continue processing this exception,
       executing any other :meth:`process_exception` methods of installed middleware,
@@ -149,19 +149,19 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
       method chain of installed middleware is started, and Scrapy won't bother calling
       any other :meth:`process_exception` methods of middleware.
 
-      If it returns a :class:`~scrapy.http.Request` object, the returned request is
+      If it returns a :class:`~scrapy.Request` object, the returned request is
       rescheduled to be downloaded in the future. This stops the execution of
       :meth:`process_exception` methods of the middleware the same as returning a
       response would.
 
       :param request: the request that generated the exception
-      :type request: is a :class:`~scrapy.http.Request` object
+      :type request: is a :class:`~scrapy.Request` object
 
       :param exception: the raised exception
       :type exception: an ``Exception`` object
 
       :param spider: the spider for which this request is intended
-      :type spider: :class:`~scrapy.spiders.Spider` object
+      :type spider: :class:`~scrapy.Spider` object
 
    .. method:: from_crawler(cls, crawler)
 
@@ -203,13 +203,13 @@ CookiesMiddleware
    browsers do.
 
    .. caution:: When non-UTF8 encoded byte sequences are passed to a
-      :class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log
+      :class:`~scrapy.Request`, the ``CookiesMiddleware`` will log
       a warning. Refer to :ref:`topics-logging-advanced-customization`
       to customize the logging behaviour.
 
    .. caution:: Cookies set via the ``Cookie`` header are not considered by the
       :ref:`cookies-mw`. If you need to set cookies for a request, use the
-      :class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
+      :class:`Request.cookies <scrapy.Request>` parameter. This is a known
       current limitation that is being worked on.
 
 The following settings can be used to configure the cookie middleware:
@@ -258,7 +258,7 @@ web server and received cookies in :class:`~scrapy.http.Response` will
 **not** be merged with the existing cookies.
 
 For more detailed information see the ``cookies`` parameter in
-:class:`~scrapy.http.Request`.
+:class:`~scrapy.Request`.
 
 .. setting:: COOKIES_DEBUG
 
@@ -501,7 +501,7 @@ defines the methods described below.
       the :signal:`open_spider <spider_opened>` signal.
 
       :param spider: the spider which has been opened
-      :type spider: :class:`~scrapy.spiders.Spider` object
+      :type spider: :class:`~scrapy.Spider` object
 
     .. method:: close_spider(spider)
 
@@ -509,27 +509,27 @@ defines the methods described below.
       the :signal:`close_spider <spider_closed>` signal.
 
       :param spider: the spider which has been closed
-      :type spider: :class:`~scrapy.spiders.Spider` object
+      :type spider: :class:`~scrapy.Spider` object
 
     .. method:: retrieve_response(spider, request)
 
       Return response if present in cache, or ``None`` otherwise.
 
       :param spider: the spider which generated the request
-      :type spider: :class:`~scrapy.spiders.Spider` object
+      :type spider: :class:`~scrapy.Spider` object
 
       :param request: the request to find cached response for
-      :type request: :class:`~scrapy.http.Request` object
+      :type request: :class:`~scrapy.Request` object
 
     .. method:: store_response(spider, request, response)
 
       Store the given response in the cache.
 
       :param spider: the spider for which the response is intended
-      :type spider: :class:`~scrapy.spiders.Spider` object
+      :type spider: :class:`~scrapy.Spider` object
 
       :param request: the corresponding request the spider generated
-      :type request: :class:`~scrapy.http.Request` object
+      :type request: :class:`~scrapy.Request` object
 
       :param response: the response to store in the cache
       :type response: :class:`~scrapy.http.Response` object
@@ -722,7 +722,7 @@ HttpProxyMiddleware
 .. class:: HttpProxyMiddleware
 
    This middleware sets the HTTP proxy to use for requests, by setting the
-   ``proxy`` meta value for :class:`~scrapy.http.Request` objects.
+   ``proxy`` meta value for :class:`~scrapy.Request` objects.
 
    Like the Python standard library module :mod:`urllib.request`, it obeys
    the following environment variables:
@@ -749,12 +749,12 @@ RedirectMiddleware
 .. reqmeta:: redirect_urls
 
 The urls which the request goes through (while being redirected) can be found
-in the ``redirect_urls`` :attr:`Request.meta <scrapy.http.Request.meta>` key.
+in the ``redirect_urls`` :attr:`Request.meta <scrapy.Request.meta>` key.
 
 .. reqmeta:: redirect_reasons
 
 The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the
-``redirect_reasons`` :attr:`Request.meta <scrapy.http.Request.meta>` key. For
+``redirect_reasons`` :attr:`Request.meta <scrapy.Request.meta>` key. For
 example: ``[301, 302, 307, 'meta refresh']``.
 
 The format of a reason depends on the middleware that handled the corresponding
@@ -770,7 +770,7 @@ settings (see the settings documentation for more info):
 
 .. reqmeta:: dont_redirect
 
-If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_redirect``
+If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_redirect``
 key set to True, the request will be ignored by this middleware.
 
 If you want to handle some redirect status codes in your spider, you can
@@ -783,7 +783,7 @@ responses (and pass them through to your spider) you can do this::
         handle_httpstatus_list = [301, 302]
 
 The ``handle_httpstatus_list`` key of :attr:`Request.meta
-<scrapy.http.Request.meta>` can also be used to specify which response codes to
+<scrapy.Request.meta>` can also be used to specify which response codes to
 allow on a per-request basis. You can also set the meta key
 ``handle_httpstatus_all`` to ``True`` if you want to allow any response code
 for a request.
@@ -889,7 +889,7 @@ settings (see the settings documentation for more info):
 
 .. reqmeta:: dont_retry
 
-If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_retry`` key
+If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_retry`` key
 set to True, the request will be ignored by this middleware.
 
 To retry requests from a spider callback, you can use the
@@ -919,7 +919,7 @@ Default: ``2``
 Maximum number of times to retry, in addition to the first download.
 
 Maximum number of retries can also be specified per-request using
-:reqmeta:`max_retry_times` attribute of :attr:`Request.meta <scrapy.http.Request.meta>`.
+:reqmeta:`max_retry_times` attribute of :attr:`Request.meta <scrapy.Request.meta>`.
 When initialized, the :reqmeta:`max_retry_times` meta key takes higher
 precedence over the :setting:`RETRY_TIMES` setting.
 
@@ -986,7 +986,7 @@ RobotsTxtMiddleware
 
 .. reqmeta:: dont_obey_robotstxt
 
-If :attr:`Request.meta <scrapy.http.Request.meta>` has
+If :attr:`Request.meta <scrapy.Request.meta>` has
 ``dont_obey_robotstxt`` key set to True
 the request will be ignored by this middleware even if
 :setting:`ROBOTSTXT_OBEY` is enabled.
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 495111b56..aa32c8943 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -62,9 +62,9 @@ download the webpage with an HTTP client like curl_ or wget_ and see if the
 information can be found in the response they get.
 
 If they get a response with the desired data, modify your Scrapy
-:class:`~scrapy.http.Request` to match that of the other HTTP client. For
+:class:`~scrapy.Request` to match that of the other HTTP client. For
 example, try using the same user-agent string (:setting:`USER_AGENT`) or the
-same :attr:`~scrapy.http.Request.headers`.
+same :attr:`~scrapy.Request.headers`.
 
 If they also get a response without the desired data, you’ll need to take
 steps to make your request more similar to that of the web browser. See
@@ -81,14 +81,14 @@ Use the :ref:`network tool <topics-network-tool>` of your web browser to see
 how your web browser performs the desired request, and try to reproduce that
 request with Scrapy.
 
-It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP
+It might be enough to yield a :class:`~scrapy.Request` with the same HTTP
 method and URL. However, you may also need to reproduce the body, headers and
-form parameters (see :class:`~scrapy.http.FormRequest`) of that request.
+form parameters (see :class:`~scrapy.FormRequest`) of that request.
 
 As all major browsers allow to export the requests in `cURL
 <https://curl.haxx.se/>`_ format, Scrapy incorporates the method
-:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent
-:class:`~scrapy.http.Request` from a cURL command. To get more information
+:meth:`~scrapy.Request.from_curl()` to generate an equivalent
+:class:`~scrapy.Request` from a cURL command. To get more information
 visit :ref:`request from curl <requests-from-curl>` inside the network
 tool section.
 
@@ -125,7 +125,7 @@ data from it depends on the type of response:
 
     If the desired data is inside HTML or XML code embedded within JSON data,
     you can load that HTML or XML code into a
-    :class:`~scrapy.selector.Selector` and then
+    :class:`~scrapy.Selector` and then
     :ref:`use it <topics-selectors>` as usual::
 
         selector = Selector(data['html'])
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 8648daded..8c30122b6 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -90,7 +90,7 @@ described next.
 1. Declaring a serializer in the field
 --------------------------------------
 
-If you use :class:`~.Item` you can declare a serializer in the
+If you use :class:`~scrapy.Item` you can declare a serializer in the
 :ref:`field metadata <topics-items-fields>`. The serializer must be
 a callable which receives a value and returns its serialized form.
 
@@ -172,7 +172,7 @@ BaseItemExporter
       :param field: the field being serialized. If the source :ref:`item object
           <item-types>` does not define field metadata, *field* is an empty
           :class:`dict`.
-      :type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance
+      :type field: :class:`~scrapy.Field` object or a :class:`dict` instance
 
       :param name: the name of the field being serialized
       :type name: str
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index af7fce852..216a8bc52 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -674,9 +674,9 @@ The function signature should be as follows:
    :type params: dict
 
    :param spider: source spider of the feed items
-   :type spider: scrapy.spiders.Spider
+   :type spider: scrapy.Spider
 
-For example, to include the :attr:`name <scrapy.spiders.Spider.name>` of the
+For example, to include the :attr:`name <scrapy.Spider.name>` of the
 source spider in the feed URI:
 
 #.  Define the following function somewhere in your project::
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 6287ee0ad..5351a2293 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -42,7 +42,7 @@ Each item pipeline component is a Python class that must implement the following
    :type item: :ref:`item object <item-types>`
 
    :param spider: the spider which scraped the item
-   :type spider: :class:`~scrapy.spiders.Spider` object
+   :type spider: :class:`~scrapy.Spider` object
 
 Additionally, they may also implement the following methods:
 
@@ -51,14 +51,14 @@ Additionally, they may also implement the following methods:
    This method is called when the spider is opened.
 
    :param spider: the spider which was opened
-   :type spider: :class:`~scrapy.spiders.Spider` object
+   :type spider: :class:`~scrapy.Spider` object
 
 .. method:: close_spider(self, spider)
 
    This method is called when the spider is closed.
 
    :param spider: the spider which was closed
-   :type spider: :class:`~scrapy.spiders.Spider` object
+   :type spider: :class:`~scrapy.Spider` object
 
 .. method:: from_crawler(cls, crawler)
 
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 65bf156ac..7cd482d07 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -42,7 +42,8 @@ Item objects
 :class:`Item` provides a :class:`dict`-like API plus additional features that
 make it the most feature-complete item type:
 
-.. class:: Item([arg])
+.. class:: scrapy.item.Item([arg])
+.. class:: scrapy.Item([arg])
 
     :class:`Item` objects replicate the standard :class:`dict` API, including
     its ``__init__`` method.
@@ -199,7 +200,8 @@ It's important to note that the :class:`Field` objects used to declare the item
 do not stay assigned as class attributes. Instead, they can be accessed through
 the :attr:`Item.fields` attribute.
 
-.. class:: Field([arg])
+.. class:: scrapy.item.Field([arg])
+.. class:: scrapy.Field([arg])
 
     The :class:`Field` class is just an alias to the built-in :class:`dict` class and
     doesn't provide any extra functionality or attributes. In other words,
@@ -317,11 +319,11 @@ If that is not the desired behavior, use a deep copy instead.
 See :mod:`copy` for more information.
 
 To create a shallow copy of an item, you can either call
-:meth:`~scrapy.item.Item.copy` on an existing item
+:meth:`~scrapy.Item.copy` on an existing item
 (``product2 = product.copy()``) or instantiate your item class from an existing
 item (``product2 = Product(product)``).
 
-To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
+To create a deep copy, call :meth:`~scrapy.Item.deepcopy` instead
 (``product2 = product.deepcopy()``).
 
 
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index d855d0133..e49f37a2f 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -74,10 +74,10 @@ on cookies.
 Request serialization
 ---------------------
 
-For persistence to work, :class:`~scrapy.http.Request` objects must be
+For persistence to work, :class:`~scrapy.Request` objects must be
 serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
 values passed to their ``__init__`` method, which must be methods of the
-running :class:`~scrapy.spiders.Spider` class.
+running :class:`~scrapy.Spider` class.
 
 If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index b895b95cb..477652704 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -27,7 +27,7 @@ Common causes of memory leaks
 
 It happens quite often (sometimes by accident, sometimes on purpose) that the
 Scrapy developer passes objects referenced in Requests (for example, using the
-:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta`
+:attr:`~scrapy.Request.cb_kwargs` or :attr:`~scrapy.Request.meta`
 attributes or the request callback function) and that effectively bounds the
 lifetime of those referenced objects to the lifetime of the Request. This is,
 by far, the most common cause of memory leaks in Scrapy projects, and a quite
@@ -48,9 +48,9 @@ Too Many Requests?
 ------------------
 
 By default Scrapy keeps the request queue in memory; it includes
-:class:`~scrapy.http.Request` objects and all objects
-referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs`
-and :attr:`~scrapy.http.Request.meta`).
+:class:`~scrapy.Request` objects and all objects
+referenced in Request attributes (e.g. in :attr:`~scrapy.Request.cb_kwargs`
+and :attr:`~scrapy.Request.meta`).
 While not necessarily a leak, this can take a lot of memory. Enabling
 :ref:`persistent job queue <topics-jobs>` could help keeping memory usage
 in control.
@@ -90,11 +90,11 @@ Which objects are tracked?
 The objects tracked by ``trackrefs`` are all from these classes (and all its
 subclasses):
 
-* :class:`scrapy.http.Request`
+* :class:`scrapy.Request`
 * :class:`scrapy.http.Response`
-* :class:`scrapy.item.Item`
-* :class:`scrapy.selector.Selector`
-* :class:`scrapy.spiders.Spider`
+* :class:`scrapy.Item`
+* :class:`scrapy.Selector`
+* :class:`scrapy.Spider`
 
 A real example
 --------------
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 00806392a..dda04dc4d 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -93,7 +93,7 @@ path::
 Logging from Spiders
 ====================
 
-Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider
+Scrapy provides a :data:`~scrapy.Spider.logger` within each Spider
 instance, which can be accessed and used like this::
 
     import scrapy
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 156897274..3438cb637 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -259,7 +259,7 @@ respectively), the pipeline will put the results under the respective field
 When using :ref:`item types <item-types>` for which fields are defined beforehand,
 you must define both the URLs field and the results field. For example, when
 using the images pipeline, items must define both the ``image_urls`` and the
-``images`` field. For instance, using the :class:`~scrapy.item.Item` class::
+``images`` field. For instance, using the :class:`~scrapy.Item` class::
 
     import scrapy
 
@@ -424,7 +424,7 @@ See here the methods that you can override in your custom Files Pipeline:
       In addition to ``response``, this method receives the original
       :class:`request <scrapy.Request>`,
       :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and 
-      :class:`item <scrapy.item.Item>`
+      :class:`item <scrapy.Item>`
 
       You can override this method to customize the download path of each file.
 
@@ -563,7 +563,7 @@ See here the methods that you can override in your custom Images Pipeline:
       In addition to ``response``, this method receives the original
       :class:`request <scrapy.Request>`,
       :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and 
-      :class:`item <scrapy.item.Item>`
+      :class:`item <scrapy.Item>`
 
       You can override this method to customize the download path of each file.
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 73b5a858f..a6a3daf31 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -35,7 +35,7 @@ Request objects
        request (once it's downloaded) as its first parameter. For more information
        see :ref:`topics-request-response-ref-request-callback-arguments` below.
        If a Request doesn't specify a callback, the spider's
-       :meth:`~scrapy.spiders.Spider.parse` method will be used.
+       :meth:`~scrapy.Spider.parse` method will be used.
        Note that if exceptions are raised during processing, errback is called instead.
 
     :type callback: collections.abc.Callable
@@ -60,7 +60,7 @@ Request objects
 
         .. caution:: Cookies set via the ``Cookie`` header are not considered by the
             :ref:`cookies-mw`. If you need to set cookies for a request, use the
-            :class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
+            :class:`Request.cookies <scrapy.Request>` parameter. This is a known
             current limitation that is being worked on.
 
     :type headers: dict
@@ -92,7 +92,7 @@ Request objects
 
         To create a request that does not send stored cookies and does not
         store received cookies, set the ``dont_merge_cookies`` key to ``True``
-        in :attr:`request.meta <scrapy.http.Request.meta>`.
+        in :attr:`request.meta <scrapy.Request.meta>`.
 
         Example of a request that sends manually-defined cookies and ignores
         cookie storage::
@@ -107,7 +107,7 @@ Request objects
 
         .. caution:: Cookies set via the ``Cookie`` header are not considered by the
             :ref:`cookies-mw`. If you need to set cookies for a request, use the
-            :class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
+            :class:`Request.cookies <scrapy.Request>` parameter. This is a known
             current limitation that is being worked on.
 
     :type cookies: dict or list
@@ -495,7 +495,9 @@ fields with form data from :class:`Response` objects.
 
 .. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms
 
-.. class:: FormRequest(url, [formdata, ...])
+.. class:: scrapy.http.request.form.FormRequest
+.. class:: scrapy.http.FormRequest
+.. class:: scrapy.FormRequest(url, [formdata, ...])
 
     The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
     remaining arguments are the same as for the :class:`Request` class and are
@@ -694,7 +696,7 @@ Response objects
 
     :param request: the initial value of the :attr:`Response.request` attribute.
         This represents the :class:`Request` that generated this response.
-    :type request: scrapy.http.Request
+    :type request: scrapy.Request
 
     :param certificate: an object representing the server's SSL certificate.
     :type certificate: twisted.internet.ssl.Certificate
@@ -920,7 +922,7 @@ TextResponse objects
 
     .. attribute:: TextResponse.selector
 
-        A :class:`~scrapy.selector.Selector` instance using the response as
+        A :class:`~scrapy.Selector` instance using the response as
         target. The selector is lazily instantiated on first access.
 
     :class:`TextResponse` objects support the following methods in addition to
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 9caba5ee5..574d4568c 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -48,7 +48,7 @@ Constructing selectors
 
 .. highlight:: python
 
-Response objects expose a :class:`~scrapy.selector.Selector` instance
+Response objects expose a :class:`~scrapy.Selector` instance
 on ``.selector`` attribute:
 
 >>> response.selector.xpath('//span/text()').get()
@@ -62,7 +62,7 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:
 >>> response.css('span::text').get()
 'good'
 
-Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
+Scrapy selectors are instances of :class:`~scrapy.Selector` class
 constructed by passing either :class:`~scrapy.http.TextResponse` object or
 markup as a string (in ``text`` argument).
 
@@ -175,7 +175,7 @@ of ``None``:
 'not-found'
 
 Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
-using ``.attrib`` property of a :class:`~scrapy.selector.Selector`:
+using ``.attrib`` property of a :class:`~scrapy.Selector`:
 
 >>> [img.attrib['src'] for img in response.css('img')]
 ['image1_thumb.jpg',
@@ -383,7 +383,7 @@ ID, or when selecting an unique element on a page):
 Using selectors with regular expressions
 ----------------------------------------
 
-:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
+:class:`~scrapy.Selector` also has a ``.re()`` method for extracting
 data using regular expressions. However, unlike using ``.xpath()`` or
 ``.css()`` methods, ``.re()`` returns a list of strings. So you
 can't construct nested ``.re()`` calls.
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0b290598f..1290b4a5e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -67,7 +67,7 @@ Example::
 
 Spiders (See the :ref:`topics-spiders` chapter for reference) can define their
 own settings that will take precedence and override the project ones. They can
-do so by setting their :attr:`~scrapy.spiders.Spider.custom_settings` attribute::
+do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute::
 
     class MySpider(scrapy.Spider):
         name = 'myspider'
@@ -142,7 +142,7 @@ In a spider, the settings are available through ``self.settings``::
     The ``settings`` attribute is set in the base Spider class after the spider
     is initialized.  If you want to use the settings before the initialization
     (e.g., in your spider's ``__init__()`` method), you'll need to override the
-    :meth:`~scrapy.spiders.Spider.from_crawler` method.
+    :meth:`~scrapy.Spider.from_crawler` method.
 
 Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
 attribute of the Crawler that is passed to ``from_crawler`` method in
@@ -338,7 +338,7 @@ is non-zero, download delay is enforced per IP, not per domain.
 DEFAULT_ITEM_CLASS
 ------------------
 
-Default: ``'scrapy.item.Item'``
+Default: ``'scrapy.Item'``
 
 The default class that will be used for instantiating items in the :ref:`the
 Scrapy shell <topics-shell>`.
@@ -360,7 +360,7 @@ The default headers used for Scrapy HTTP Requests. They're populated in the
 
 .. caution:: Cookies set via the ``Cookie`` header are not considered by the
     :ref:`cookies-mw`. If you need to set cookies for a request, use the
-    :class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
+    :class:`Request.cookies <scrapy.Request>` parameter. This is a known
     current limitation that is being worked on.
 
 .. setting:: DEPTH_LIMIT
@@ -384,8 +384,8 @@ Default: ``0``
 
 Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
 
-An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of
-a :class:`~scrapy.http.Request` based on its depth.
+An integer that is used to adjust the :attr:`~scrapy.Request.priority` of
+a :class:`~scrapy.Request` based on its depth.
 
 The priority of a request is adjusted as follows::
 
@@ -816,14 +816,14 @@ The default (``RFPDupeFilter``) filters based on request fingerprint using
 the ``scrapy.utils.request.request_fingerprint`` function. In order to change
 the way duplicates are checked you could subclass ``RFPDupeFilter`` and
 override its ``request_fingerprint`` method. This method should accept
-scrapy :class:`~scrapy.http.Request` object and return its fingerprint
+scrapy :class:`~scrapy.Request` object and return its fingerprint
 (a string).
 
 You can disable filtering of duplicate requests by setting
 :setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
 Be very careful about this however, because you can get into crawling loops.
 It's usually a better idea to set the ``dont_filter`` parameter to
-``True`` on the specific :class:`~scrapy.http.Request` that should not be
+``True`` on the specific :class:`~scrapy.Request` that should not be
 filtered.
 
 .. setting:: DUPEFILTER_DEBUG
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index b910fc453..8c90a506c 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -118,7 +118,7 @@ Available Scrapy objects
 
 The Scrapy shell automatically creates some convenient objects from the
 downloaded page, like the :class:`~scrapy.http.Response` object and the
-:class:`~scrapy.selector.Selector` objects (for both HTML and XML
+:class:`~scrapy.Selector` objects (for both HTML and XML
 content).
 
 Those objects are:
@@ -126,12 +126,12 @@ Those objects are:
 -    ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object.
 
 -   ``spider`` - the Spider which is known to handle the URL, or a
-    :class:`~scrapy.spiders.Spider` object if there is no spider found for the
+    :class:`~scrapy.Spider` object if there is no spider found for the
     current URL
 
--   ``request`` - a :class:`~scrapy.http.Request` object of the last fetched
+-   ``request`` - a :class:`~scrapy.Request` object of the last fetched
     page. You can modify this request using
-    :meth:`~scrapy.http.Request.replace` or fetch a new request (without
+    :meth:`~scrapy.Request.replace` or fetch a new request (without
     leaving the shell) using the ``fetch`` shortcut.
 
 -   ``response`` - a :class:`~scrapy.http.Response` object containing the last
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 530af1e37..a67cc1879 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -155,7 +155,7 @@ item_scraped
     :type item: :ref:`item object <item-types>`
 
     :param spider: the spider which scraped the item
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
     :param response: the response from where the item was scraped
     :type response: :class:`~scrapy.http.Response` object
@@ -175,7 +175,7 @@ item_dropped
     :type item: :ref:`item object <item-types>`
 
     :param spider: the spider which scraped the item
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
     :param response: the response from where the item was dropped
     :type response: :class:`~scrapy.http.Response` object
@@ -203,7 +203,7 @@ item_error
     :type response: :class:`~scrapy.http.Response` object
 
     :param spider: the spider which raised the exception
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
     :param failure: the exception raised
     :type failure: twisted.python.failure.Failure
@@ -223,7 +223,7 @@ spider_closed
     This signal supports returning deferreds from its handlers.
 
     :param spider: the spider which has been closed
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
     :param reason: a string which describes the reason why the spider was closed. If
         it was closed because the spider has completed scraping, the reason
@@ -247,7 +247,7 @@ spider_opened
     This signal supports returning deferreds from its handlers.
 
     :param spider: the spider which has been opened
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
 spider_idle
 ~~~~~~~~~~~
@@ -278,7 +278,7 @@ spider_idle
     This signal does not support returning deferreds from its handlers.
 
     :param spider: the spider which has gone idle
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
 .. note:: Scheduling some requests in your :signal:`spider_idle` handler does
     **not** guarantee that it can prevent the spider from being closed,
@@ -303,7 +303,7 @@ spider_error
     :type response: :class:`~scrapy.http.Response` object
 
     :param spider: the spider which raised the exception
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
 Request signals
 ---------------
@@ -314,16 +314,16 @@ request_scheduled
 .. signal:: request_scheduled
 .. function:: request_scheduled(request, spider)
 
-    Sent when the engine schedules a :class:`~scrapy.http.Request`, to be
+    Sent when the engine schedules a :class:`~scrapy.Request`, to be
     downloaded later.
 
     This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the scheduler
-    :type request: :class:`~scrapy.http.Request` object
+    :type request: :class:`~scrapy.Request` object
 
     :param spider: the spider that yielded the request
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
 request_dropped
 ~~~~~~~~~~~~~~~
@@ -331,16 +331,16 @@ request_dropped
 .. signal:: request_dropped
 .. function:: request_dropped(request, spider)
 
-    Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
+    Sent when a :class:`~scrapy.Request`, scheduled by the engine to be
     downloaded later, is rejected by the scheduler.
 
     This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the scheduler
-    :type request: :class:`~scrapy.http.Request` object
+    :type request: :class:`~scrapy.Request` object
 
     :param spider: the spider that yielded the request
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
 request_reached_downloader
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -348,15 +348,15 @@ request_reached_downloader
 .. signal:: request_reached_downloader
 .. function:: request_reached_downloader(request, spider)
 
-    Sent when a :class:`~scrapy.http.Request` reached downloader.
+    Sent when a :class:`~scrapy.Request` reached downloader.
 
     This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached downloader
-    :type request: :class:`~scrapy.http.Request` object
+    :type request: :class:`~scrapy.Request` object
 
     :param spider: the spider that yielded the request
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
 request_left_downloader
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -366,16 +366,16 @@ request_left_downloader
 
     .. versionadded:: 2.0
 
-    Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of
+    Sent when a :class:`~scrapy.Request` leaves the downloader, even in case of
     failure.
 
     This signal does not support returning deferreds from its handlers.
 
     :param request: the request that reached the downloader
-    :type request: :class:`~scrapy.http.Request` object
+    :type request: :class:`~scrapy.Request` object
 
     :param spider: the spider that yielded the request
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
 bytes_received
 ~~~~~~~~~~~~~~
@@ -402,10 +402,10 @@ bytes_received
     :type data: :class:`bytes` object
 
     :param request: the request that generated the download
-    :type request: :class:`~scrapy.http.Request` object
+    :type request: :class:`~scrapy.Request` object
 
     :param spider: the spider associated with the response
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
 headers_received
 ~~~~~~~~~~~~~~~~
@@ -432,10 +432,10 @@ headers_received
     :type body_length: `int`
 
     :param request: the request that generated the download
-    :type request: :class:`~scrapy.http.Request` object
+    :type request: :class:`~scrapy.Request` object
 
     :param spider: the spider associated with the response
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
 Response signals
 ----------------
@@ -455,10 +455,10 @@ response_received
     :type response: :class:`~scrapy.http.Response` object
 
     :param request: the request that generated the response
-    :type request: :class:`~scrapy.http.Request` object
+    :type request: :class:`~scrapy.Request` object
 
     :param spider: the spider for which the response is intended
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
 
 .. note:: The ``request`` argument might not contain the original request that
     reached the downloader, if a :ref:`topics-downloader-middleware` modifies
@@ -479,7 +479,7 @@ response_downloaded
     :type response: :class:`~scrapy.http.Response` object
 
     :param request: the request that generated the response
-    :type request: :class:`~scrapy.http.Request` object
+    :type request: :class:`~scrapy.Request` object
 
     :param spider: the spider for which the response is intended
-    :type spider: :class:`~scrapy.spiders.Spider` object
+    :type spider: :class:`~scrapy.Spider` object
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 11bbbb58d..f0158dc41 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -93,7 +93,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         :type response: :class:`~scrapy.http.Response` object
 
         :param spider: the spider for which this response is intended
-        :type spider: :class:`~scrapy.spiders.Spider` object
+        :type spider: :class:`~scrapy.Spider` object
 
 
     .. method:: process_spider_output(response, result, spider)
@@ -102,7 +102,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         it has processed the response.
 
         :meth:`process_spider_output` must return an iterable of
-        :class:`~scrapy.http.Request` objects and :ref:`item object
+        :class:`~scrapy.Request` objects and :ref:`item object
         <topics-items>`.
 
         :param response: the response which generated this output from the
@@ -110,11 +110,11 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         :type response: :class:`~scrapy.http.Response` object
 
         :param result: the result returned by the spider
-        :type result: an iterable of :class:`~scrapy.http.Request` objects and
+        :type result: an iterable of :class:`~scrapy.Request` objects and
           :ref:`item object <topics-items>`
 
         :param spider: the spider whose result is being processed
-        :type spider: :class:`~scrapy.spiders.Spider` object
+        :type spider: :class:`~scrapy.Spider` object
 
     .. method:: process_spider_exception(response, exception, spider)
 
@@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.http.Request` objects and :ref:`item object
+        iterable of :class:`~scrapy.Request` objects and :ref:`item object
         <topics-items>`.
 
         If it returns ``None``, Scrapy will continue processing this exception,
@@ -142,7 +142,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         :type exception: :exc:`Exception` object
 
         :param spider: the spider which raised the exception
-        :type spider: :class:`~scrapy.spiders.Spider` object
+        :type spider: :class:`~scrapy.Spider` object
 
     .. method:: process_start_requests(start_requests, spider)
 
@@ -152,7 +152,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         items).
 
         It receives an iterable (in the ``start_requests`` parameter) and must
-        return another iterable of :class:`~scrapy.http.Request` objects.
+        return another iterable of :class:`~scrapy.Request` objects.
 
         .. note:: When implementing this method in your spider middleware, you
            should always return an iterable (that follows the input one) and
@@ -164,10 +164,10 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
            (like a time limit or item/page count).
 
         :param start_requests: the start requests
-        :type start_requests: an iterable of :class:`~scrapy.http.Request`
+        :type start_requests: an iterable of :class:`~scrapy.Request`
 
         :param spider: the spider to whom the start requests belong
-        :type spider: :class:`~scrapy.spiders.Spider` object
+        :type spider: :class:`~scrapy.Spider` object
 
     .. method:: from_crawler(cls, crawler)
 
@@ -251,7 +251,7 @@ this::
 .. reqmeta:: handle_httpstatus_all
 
 The ``handle_httpstatus_list`` key of :attr:`Request.meta
-<scrapy.http.Request.meta>` can also be used to specify which response codes to
+<scrapy.Request.meta>` can also be used to specify which response codes to
 allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all``
 to ``True`` if you want to allow any response code for a request, and ``False`` to
 disable the effects of the ``handle_httpstatus_all`` key.
@@ -295,7 +295,7 @@ OffsiteMiddleware
    Filters out Requests for URLs outside the domains covered by the spider.
 
    This middleware filters out every request whose host names aren't in the
-   spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute.
+   spider's :attr:`~scrapy.Spider.allowed_domains` attribute.
    All subdomains of any domain in the list are also allowed.
    E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
    but not ``www2.example.com`` nor ``example.com``.
@@ -313,10 +313,10 @@ OffsiteMiddleware
    will be printed (but only for the first request filtered).
 
    If the spider doesn't define an
-   :attr:`~scrapy.spiders.Spider.allowed_domains` attribute, or the
+   :attr:`~scrapy.Spider.allowed_domains` attribute, or the
    attribute is empty, the offsite middleware will allow all requests.
 
-   If the request has the :attr:`~scrapy.http.Request.dont_filter` attribute
+   If the request has the :attr:`~scrapy.Request.dont_filter` attribute
    set, the offsite middleware will allow the request even if its domain is not
    listed in allowed domains.
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 903fbd383..67b9e2e0e 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -17,15 +17,15 @@ For spiders, the scraping cycle goes through something like this:
    those requests.
 
    The first requests to perform are obtained by calling the
-   :meth:`~scrapy.spiders.Spider.start_requests` method which (by default)
-   generates :class:`~scrapy.http.Request` for the URLs specified in the
-   :attr:`~scrapy.spiders.Spider.start_urls` and the
-   :attr:`~scrapy.spiders.Spider.parse` method as callback function for the
+   :meth:`~scrapy.Spider.start_requests` method which (by default)
+   generates :class:`~scrapy.Request` for the URLs specified in the
+   :attr:`~scrapy.Spider.start_urls` and the
+   :attr:`~scrapy.Spider.parse` method as callback function for the
    Requests.
 
 2. In the callback function, you parse the response (web page) and return
    :ref:`item objects <topics-items>`,
-   :class:`~scrapy.http.Request` objects, or an iterable of these objects.
+   :class:`~scrapy.Request` objects, or an iterable of these objects.
    Those Requests will also contain a callback (maybe
    the same) and will then be downloaded by Scrapy and then their
    response handled by the specified callback.
@@ -50,7 +50,8 @@ We will talk about those types here.
 scrapy.Spider
 =============
 
-.. class:: Spider()
+.. class:: scrapy.spiders.Spider()
+.. class:: scrapy.Spider()
 
    This is the simplest spider, and the one from which every other spider
    must inherit (including spiders that come bundled with Scrapy, as well as spiders
@@ -86,7 +87,7 @@ scrapy.Spider
 
        A list of URLs where the spider will begin to crawl from, when no
        particular URLs are specified. So, the first pages downloaded will be those
-       listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data
+       listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data
        contained in the start URLs.
 
    .. attribute:: custom_settings
@@ -179,7 +180,7 @@ scrapy.Spider
        the same requirements as the :class:`Spider` class.
 
        This method, as well as any other Request callback, must return an
-       iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects
+       iterable of :class:`~scrapy.Request` and/or :ref:`item objects
        <topics-items>`.
 
        :param response: the response to parse
@@ -234,7 +235,7 @@ Return multiple Requests and items from a single callback::
                 yield scrapy.Request(response.urljoin(href), self.parse)
 
 Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
-to give data more structure you can use :class:`~scrapy.item.Item` objects::
+to give data more structure you can use :class:`~scrapy.Item` objects::
 
     import scrapy
     from myproject.items import MyItem
@@ -373,7 +374,7 @@ CrawlSpider
       This method is called for each response produced for the URLs in
       the spider's ``start_urls`` attribute. It allows to parse
       the initial responses and must return either an
-      :ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
+      :ref:`item object <topics-items>`, a :class:`~scrapy.Request`
       object, or an iterable containing any of them.
 
 Crawling rules
@@ -383,7 +384,7 @@ Crawling rules
 
    ``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
    defines how links will be extracted from each crawled page. Each produced link will
-   be used to generate a :class:`~scrapy.http.Request` object, which will contain the
+   be used to generate a :class:`~scrapy.Request` object, which will contain the
    link's text in its ``meta`` dictionary (under the ``link_text`` key).
    If omitted, a default link extractor created with no arguments will be used,
    resulting in all links being extracted.
@@ -392,9 +393,9 @@ Crawling rules
    object with that name will be used) to be called for each link extracted with
    the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
    as its first argument and must return either a single instance or an iterable of
-   :ref:`item objects <topics-items>` and/or :class:`~scrapy.http.Request` objects
+   :ref:`item objects <topics-items>` and/or :class:`~scrapy.Request` objects
    (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
-   object will contain the text of the link that produced the :class:`~scrapy.http.Request`
+   object will contain the text of the link that produced the :class:`~scrapy.Request`
    in its ``meta`` dictionary (under the ``link_text`` key)
 
    ``cb_kwargs`` is a dict containing the keyword arguments to be passed to the
@@ -411,7 +412,7 @@ Crawling rules
 
    ``process_request`` is a callable (or a string, in which case a method from
    the spider object with that name will be used) which will be called for every
-   :class:`~scrapy.http.Request` extracted by this rule. This callable should
+   :class:`~scrapy.Request` extracted by this rule. This callable should
    take said request as first argument and the :class:`~scrapy.http.Response`
    from which the request originated as second argument. It must return a
    ``Request`` object or ``None`` (to filter out the request).
@@ -470,7 +471,7 @@ Let's now take a look at an example CrawlSpider with rules::
 This spider would start crawling example.com's home page, collecting category
 links, and item links, parsing the latter with the ``parse_item`` method. For
 each item response, some data will be extracted from the HTML using XPath, and
-an :class:`~scrapy.item.Item` will be filled with it.
+an :class:`~scrapy.Item` will be filled with it.
 
 XMLFeedSpider
 -------------
@@ -493,11 +494,11 @@ XMLFeedSpider
 
            - ``'iternodes'`` - a fast iterator based on regular expressions
 
-           - ``'html'`` - an iterator which uses :class:`~scrapy.selector.Selector`.
+           - ``'html'`` - an iterator which uses :class:`~scrapy.Selector`.
              Keep in mind this uses DOM parsing and must load all DOM in memory
              which could be a problem for big feeds
 
-           - ``'xml'`` - an iterator which uses :class:`~scrapy.selector.Selector`.
+           - ``'xml'`` - an iterator which uses :class:`~scrapy.Selector`.
              Keep in mind this uses DOM parsing and must load all DOM in memory
              which could be a problem for big feeds
 
@@ -515,7 +516,7 @@ XMLFeedSpider
         available in that document that will be processed with this spider. The
         ``prefix`` and ``uri`` will be used to automatically register
         namespaces using the
-        :meth:`~scrapy.selector.Selector.register_namespace` method.
+        :meth:`~scrapy.Selector.register_namespace` method.
 
         You can then specify nodes with namespaces in the :attr:`itertag`
         attribute.
@@ -542,10 +543,10 @@ XMLFeedSpider
 
         This method is called for the nodes matching the provided tag name
         (``itertag``).  Receives the response and an
-        :class:`~scrapy.selector.Selector` for each node.  Overriding this
+        :class:`~scrapy.Selector` for each node.  Overriding this
         method is mandatory. Otherwise, you spider won't work.  This method
         must return an :ref:`item object <topics-items>`, a
-        :class:`~scrapy.http.Request` object, or an iterable containing any of
+        :class:`~scrapy.Request` object, or an iterable containing any of
         them.
 
     .. method:: process_results(response, results)
@@ -587,7 +588,7 @@ These spiders are pretty easy to use, let's have a look at one example::
 
 Basically what we did up there was to create a spider that downloads a feed from
 the given ``start_urls``, and then iterates through each of its ``item`` tags,
-prints them out, and stores some random data in an :class:`~scrapy.item.Item`.
+prints them out, and stores some random data in an :class:`~scrapy.Item`.
 
 CSVFeedSpider
 -------------

From bcce0660573cf9309e3decf853488da2ef8bc576 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Wed, 14 Jul 2021 12:56:07 -0300
Subject: [PATCH 412/716] Update ItemFilter (#5203)

---
 scrapy/extensions/feedexport.py | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 84a79e32d..bd4808e2b 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -11,6 +11,7 @@ import sys
 import warnings
 from datetime import datetime
 from tempfile import NamedTemporaryFile
+from typing import Any, Optional, Tuple
 from urllib.parse import unquote, urlparse
 
 from twisted.internet import defer, threads
@@ -54,16 +55,19 @@ class ItemFilter:
     :param feed_options: feed specific options passed from FeedExporter
     :type feed_options: dict
     """
+    feed_options: Optional[dict]
+    item_classes: Tuple
 
-    def __init__(self, feed_options):
+    def __init__(self, feed_options: Optional[dict]) -> None:
         self.feed_options = feed_options
-        self.item_classes = set()
+        if feed_options is not None:
+            self.item_classes = tuple(
+                load_object(item_class) for item_class in feed_options.get("item_classes") or ()
+            )
+        else:
+            self.item_classes = tuple()
 
-        if 'item_classes' in self.feed_options:
-            for item_class in self.feed_options['item_classes']:
-                self.item_classes.add(load_object(item_class))
-
-    def accepts(self, item):
+    def accepts(self, item: Any) -> bool:
         """
         Return ``True`` if `item` should be exported or ``False`` otherwise.
 
@@ -73,9 +77,8 @@ class ItemFilter:
         :rtype: bool
         """
         if self.item_classes:
-            return isinstance(item, tuple(self.item_classes))
-
-        return True    # accept all items if none declared in item_classes
+            return isinstance(item, self.item_classes)
+        return True  # accept all items by default
 
 
 class IFeedStorage(Interface):

From 89b654b82df763a59d6a37e92e796c1478c768ce Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Fri, 16 Jul 2021 15:18:14 +0500
Subject: [PATCH 413/716] Make the pylint test pass (#5207)

Co-authored-by: Vostretsov Nikita <whalebot.helmsman@gmail.com>
---
 .github/workflows/checks.yml       | 4 ++++
 .github/workflows/tests-ubuntu.yml | 4 ++++
 pylintrc                           | 4 ++++
 3 files changed, 12 insertions(+)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index e7080db9a..6bdfcb5dc 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -19,6 +19,7 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: pylint
+            TOX_PIP_VERSION: 20.3.3
         - python-version: 3.9
           env:
             TOXENV: typing
@@ -37,5 +38,8 @@ jobs:
     - name: Run check
       env: ${{ matrix.env }}
       run: |
+        if [[ ! -z "$TOX_PIP_VERSION" ]]; then
+            pip install tox-pip-version
+        fi
         pip install -U tox
         tox
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index b42e8b127..521d7ae70 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -40,6 +40,7 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: extra-deps
+            TOX_PIP_VERSION: 20.3.3
         - python-version: 3.9
           env:
             TOXENV: asyncio
@@ -68,6 +69,9 @@ jobs:
           $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
           source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
         fi
+        if [[ ! -z "$TOX_PIP_VERSION" ]]; then
+            pip install tox-pip-version
+        fi
         pip install -U tox
         tox
 
diff --git a/pylintrc b/pylintrc
index 972bf99de..a44712507 100644
--- a/pylintrc
+++ b/pylintrc
@@ -6,6 +6,7 @@ jobs=1  # >1 hides results
 disable=abstract-method,
         anomalous-backslash-in-string,
         arguments-differ,
+        arguments-renamed,
         attribute-defined-outside-init,
         bad-classmethod-argument,
         bad-continuation,
@@ -21,6 +22,8 @@ disable=abstract-method,
         cell-var-from-loop,
         comparison-with-callable,
         consider-iterating-dictionary,
+        consider-using-dict-items,
+        consider-using-from-import,
         consider-using-in,
         consider-using-set-comprehension,
         consider-using-sys-exit,
@@ -105,6 +108,7 @@ disable=abstract-method,
         unsubscriptable-object,
         unused-argument,
         unused-import,
+        unused-private-member,
         unused-variable,
         unused-wildcard-import,
         used-before-assignment,

From ee2df97bbdf9120ccefc9c132bcbf0994479f948 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Fri, 16 Jul 2021 17:28:32 +0500
Subject: [PATCH 414/716] Pin the libxml2 version in CI as a newer one breaks
 lxml (#5208)

---
 .github/workflows/tests-ubuntu.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 521d7ae70..57188bd63 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -57,7 +57,8 @@ jobs:
       if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')
       run: |
         sudo apt-get update
-        sudo apt-get install libxml2-dev libxslt-dev
+        # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
+        sudo apt-get install libxml2-dev/bionic-updates libxslt-dev
 
     - name: Run tests
       env: ${{ matrix.env }}

From 7306a81188f81964ad85f4936ce29e3aa0084447 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 19 Jul 2021 20:09:11 +0500
Subject: [PATCH 415/716] Disable builtin middlewares in spider middleware
 tests.

---
 tests/test_spidermiddleware.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 0a6b96c0c..b0ca2f62e 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -19,7 +19,7 @@ class SpiderMiddlewareTestCase(TestCase):
     def setUp(self):
         self.request = Request('http://example.com/index.html')
         self.response = Response(self.request.url, request=self.request)
-        self.crawler = get_crawler(Spider)
+        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}})
         self.spider = self.crawler._create_spider('foo')
         self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
 

From 70dddfe2b293171db4c58511175edf55bbe1831c Mon Sep 17 00:00:00 2001
From: Pascal Corpet <pascal@bayesimpact.org>
Date: Wed, 21 Jul 2021 17:10:10 +0200
Subject: [PATCH 416/716] Typing: switch to a newer version of MyPy to check
 types

---
 tests/CrawlerProcess/asyncio_deferred_signal.py | 2 ++
 tox.ini                                         | 5 ++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py
index 46c2a12a4..bdd3c1fef 100644
--- a/tests/CrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py
@@ -1,5 +1,6 @@
 import asyncio
 import sys
+from typing import Optional
 
 from scrapy import Spider
 from scrapy.crawler import CrawlerProcess
@@ -31,6 +32,7 @@ class UrlSpider(Spider):
 
 
 if __name__ == "__main__":
+    ASYNCIO_EVENT_LOOP: Optional[str]
     try:
         ASYNCIO_EVENT_LOOP = sys.argv[1]
     except IndexError:
diff --git a/tox.ini b/tox.ini
index 8167aff96..4c4bbff6e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -35,7 +35,10 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    mypy==0.780
+    lxml-stubs==0.2.0
+    mypy==0.910
+    types-pyOpenSSL==20.0.3
+    types-setuptools==57.0.0
 commands =
     mypy --show-error-codes {posargs: scrapy tests}
 

From 209c1fce02a776b2917559c09d977827f858743a Mon Sep 17 00:00:00 2001
From: Aaron Tan <70739609+aaron-tan@users.noreply.github.com>
Date: Sat, 24 Jul 2021 14:50:48 +1000
Subject: [PATCH 417/716] Reference MailSender in StatsMailer

Added a reference to MailSender in the StatsMailer extension description and included a link to the document detailing how to instantiate MailSender and using Scrapy settings objects.
---
 docs/topics/extensions.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 9e86fd0fe..3cabcefdd 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -323,6 +323,15 @@ domain has finished scraping, including the Scrapy stats collected. The email
 will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS`
 setting.
 
+Emails can be sent using the MailSender class
+
+.. module:: scrapy.mail
+   :synopsis: MailSender class
+
+.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
+
+To see a full list of parameters, including examples on how to instantiate MailSender and using mail settings, see :ref:`topics-email`
+
 .. module:: scrapy.extensions.debug
    :synopsis: Extensions for debugging Scrapy
 

From b22a0043988a4f3c54709988de99f489db44f78d Mon Sep 17 00:00:00 2001
From: Rob Banagale <banagale@users.noreply.github.com>
Date: Mon, 26 Jul 2021 11:51:32 -0700
Subject: [PATCH 418/716] Document media pipeline file naming (#5152)

---
 docs/topics/media-pipeline.rst | 88 ++++++++++++++++++++++++++++------
 1 file changed, 74 insertions(+), 14 deletions(-)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 3438cb637..46bd2859b 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -111,25 +111,82 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
 
    IMAGES_STORE = '/path/to/valid/dir'
 
+.. _topics-file-naming:
+
+File Naming
+===========
+
+Default File Naming
+-------------------
+
+By default, files are stored using an `SHA-1 hash`_ of their URLs for the file names.
+
+For example, the following image URL::
+
+    http://www.example.com/image.jpg
+
+Whose ``SHA-1 hash`` is::
+
+    3afec3b4765f8f0a07b78f98c07b83f013567a0a
+
+Will be downloaded and stored using your chosen :ref:`storage method <topics-supported-storage>` and the following file name::
+
+   3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg
+
+Custom File Naming
+-------------------
+
+You may wish to use a different calculated file name for saved files.
+For example, classifying an image by including meta in the file name.
+
+Customize file names by overriding the ``file_path`` method of your
+media pipeline.
+
+For example, an image pipeline with image URL::
+
+   http://www.example.com/product/images/large/front/0000000004166
+
+Can be processed into a file name with a condensed hash and the perspective
+``front``::
+
+  00b08510e4_front.jpg
+
+By overriding ``file_path`` like this:
+
+.. code-block:: python
+
+  import hashlib
+  from os.path import splitext
+
+  def file_path(self, request, response=None, info=None, *, item=None):
+      image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)
+      image_perspective = request.url.split('/')[-2]
+      image_filename = f'{image_url_hash}_{image_perspective}.jpg'
+
+      return image_filename
+
+.. warning::
+  If your custom file name scheme relies on meta data that can vary between
+  scrapes it may lead to unexpected re-downloading of existing media using
+  new file names.
+
+  For example, if your custom file name scheme uses a product title and the
+  site changes an item's product title between scrapes, Scrapy will re-download
+  the same media using updated file names.
+
+For more information about the ``file_path`` method, see :ref:`topics-media-pipeline-override`.
+
+.. _topics-supported-storage:
+
 Supported Storage
 =================
 
 File system storage
 -------------------
 
-The files are stored using a `SHA1 hash`_ of their URLs for the file names.
+File system storage will save files to the following path::
 
-For example, the following image URL::
-
-    http://www.example.com/image.jpg
-
-Whose ``SHA1 hash`` is::
-
-    3afec3b4765f8f0a07b78f98c07b83f013567a0a
-
-Will be downloaded and stored in the following file::
-
-   <IMAGES_STORE>/full/3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg
+   <IMAGES_STORE>/full/<FILE_NAME>
 
 Where:
 
@@ -139,6 +196,9 @@ Where:
 * ``full`` is a sub-directory to separate full images from thumbnails (if
   used). For more info see :ref:`topics-images-thumbnails`.
 
+* ``<FILE_NAME>`` is the file name assigned to the file.  For more info see :ref:`topics-file-naming`.
+
+
 .. _media-pipeline-ftp:
 
 FTP server storage
@@ -353,9 +413,9 @@ Where:
 * ``<size_name>`` is the one specified in the :setting:`IMAGES_THUMBS`
   dictionary keys (``small``, ``big``, etc)
 
-* ``<image_id>`` is the `SHA1 hash`_ of the image url
+* ``<image_id>`` is the `SHA-1 hash`_ of the image url
 
-.. _SHA1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions
+.. _SHA-1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions
 
 Example of image files stored using ``small`` and ``big`` thumbnail names::
 

From abe0b37d307d40897ca6d7e61aa5c137c8e6a4c1 Mon Sep 17 00:00:00 2001
From: laggardkernel <laggardkernel@users.noreply.github.com>
Date: Tue, 27 Jul 2021 17:11:32 +0800
Subject: [PATCH 419/716] Cleanup leftover boto2 code in S3DownloaderHandler
 (#5209)

S3DownloaderHandler.conn is a leftover attribute from 5e99758.
---
 scrapy/core/downloader/handlers/s3.py | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 1966570d4..31f1be31a 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,5 +1,3 @@
-from urllib.parse import unquote
-
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.boto import is_botocore_available
@@ -59,7 +57,7 @@ class S3DownloadHandler:
         url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
         if self.anon:
             request = request.replace(url=url)
-        elif self._signer is not None:
+        else:
             import botocore.awsrequest
             awsrequest = botocore.awsrequest.AWSRequest(
                 method=request.method,
@@ -69,14 +67,4 @@ class S3DownloadHandler:
             self._signer.add_auth(awsrequest)
             request = request.replace(
                 url=url, headers=awsrequest.headers.items())
-        else:
-            signed_headers = self.conn.make_request(
-                method=request.method,
-                bucket=bucket,
-                key=unquote(p.path),
-                query_args=unquote(p.query),
-                headers=request.headers,
-                data=request.body,
-            )
-            request = request.replace(url=url, headers=signed_headers)
         return self._download_http(request, spider)

From 7e4321f201a795166d779f2aa0b36d38cb50106e Mon Sep 17 00:00:00 2001
From: laggardkernel <laggardkernel@gmail.com>
Date: Mon, 19 Jul 2021 12:00:42 +0800
Subject: [PATCH 420/716] Add support for temporary security credential in AWS
 auth

---
 docs/topics/feed-exports.rst          |  5 +++--
 docs/topics/settings.rst              | 14 ++++++++++++++
 scrapy/core/downloader/handlers/s3.py |  5 ++++-
 scrapy/extensions/feedexport.py       |  5 ++++-
 scrapy/pipelines/files.py             |  3 +++
 scrapy/pipelines/images.py            |  1 +
 tests/test_feedexport.py              | 12 ++++++++----
 tox.ini                               |  1 +
 8 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 216a8bc52..af60de716 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -135,7 +135,7 @@ Here are some examples to illustrate:
 
     -   ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
 
-.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence 
+.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
           they can also be used as storage URI parameters.
 
 
@@ -200,6 +200,7 @@ passed through the following settings:
 
 -   :setting:`AWS_ACCESS_KEY_ID`
 -   :setting:`AWS_SECRET_ACCESS_KEY`
+-   :setting:`AWS_SESSION_TOKEN` (Optional)
 
 You can also define a custom ACL and custom endpoint for exported feeds using this setting:
 
@@ -357,7 +358,7 @@ For instance::
             'item_export_kwargs': {
                'export_empty_fields': True,
             },
-        }, 
+        },
         '/home/user/documents/items.xml': {
             'format': 'xml',
             'fields': ['name', 'price'],
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 1290b4a5e..58daafa6f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -204,6 +204,20 @@ Default: ``None``
 The AWS secret key used by code that requires access to `Amazon Web services`_,
 such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
 
+.. setting:: AWS_SESSION_TOKEN
+
+AWS_SESSION_TOKEN
+-----------------
+
+Default: ``None`` (Optional)
+
+The AWS security token used by code that requires access to `Amazon Web services`_,
+such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
+
+The security token is only required by a *temporary security credentials*.
+Using of temporary security credentials is discouraged cause the credentials
+are short term. It may expires before the scraping is done.
+
 .. setting:: AWS_ENDPOINT_URL
 
 AWS_ENDPOINT_URL
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 31f1be31a..51ca1ed5e 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -10,6 +10,7 @@ class S3DownloadHandler:
     def __init__(self, settings, *,
                  crawler=None,
                  aws_access_key_id=None, aws_secret_access_key=None,
+                 aws_session_token=None,
                  httpdownloadhandler=HTTPDownloadHandler, **kw):
         if not is_botocore_available():
             raise NotConfigured('missing botocore library')
@@ -18,6 +19,8 @@ class S3DownloadHandler:
             aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
         if not aws_secret_access_key:
             aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
+        if not aws_session_token:
+            aws_session_token = settings['AWS_SESSION_TOKEN']
 
         # If no credentials could be found anywhere,
         # consider this an anonymous connection request by default;
@@ -36,7 +39,7 @@ class S3DownloadHandler:
         if not self.anon:
             SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
             self._signer = SignerCls(botocore.credentials.Credentials(
-                aws_access_key_id, aws_secret_access_key))
+                aws_access_key_id, aws_secret_access_key, aws_session_token))
 
         _http_handler = create_instance(
             objcls=httpdownloadhandler,
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index bd4808e2b..564c736f2 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -154,13 +154,14 @@ class FileFeedStorage:
 class S3FeedStorage(BlockingFeedStorage):
 
     def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *,
-                 feed_options=None):
+                 feed_options=None, session_token=None):
         if not is_botocore_available():
             raise NotConfigured('missing botocore library')
         u = urlparse(uri)
         self.bucketname = u.hostname
         self.access_key = u.username or access_key
         self.secret_key = u.password or secret_key
+        self.session_token = session_token
         self.keyname = u.path[1:]  # remove first "/"
         self.acl = acl
         self.endpoint_url = endpoint_url
@@ -169,6 +170,7 @@ class S3FeedStorage(BlockingFeedStorage):
         self.s3_client = session.create_client(
             's3', aws_access_key_id=self.access_key,
             aws_secret_access_key=self.secret_key,
+            aws_session_token=self.session_token,
             endpoint_url=self.endpoint_url)
         if feed_options and feed_options.get('overwrite', True) is False:
             logger.warning('S3 does not support appending to files. To '
@@ -182,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage):
             uri,
             access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
             secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
+            session_token=crawler.settings['AWS_SESSION_TOKEN'],
             acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
             endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None,
             feed_options=feed_options,
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 13ecd4e6c..8766ef66f 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -79,6 +79,7 @@ class FSFilesStore:
 class S3FilesStore:
     AWS_ACCESS_KEY_ID = None
     AWS_SECRET_ACCESS_KEY = None
+    AWS_SESSION_TOKEN = None
     AWS_ENDPOINT_URL = None
     AWS_REGION_NAME = None
     AWS_USE_SSL = None
@@ -98,6 +99,7 @@ class S3FilesStore:
             's3',
             aws_access_key_id=self.AWS_ACCESS_KEY_ID,
             aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
+            aws_session_token=self.AWS_SESSION_TOKEN,
             endpoint_url=self.AWS_ENDPOINT_URL,
             region_name=self.AWS_REGION_NAME,
             use_ssl=self.AWS_USE_SSL,
@@ -349,6 +351,7 @@ class FilesPipeline(MediaPipeline):
         s3store = cls.STORE_SCHEMES['s3']
         s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+        s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN']
         s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
         s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']
         s3store.AWS_USE_SSL = settings['AWS_USE_SSL']
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index e3ab23ea5..9c99dc69e 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -92,6 +92,7 @@ class ImagesPipeline(FilesPipeline):
         s3store = cls.STORE_SCHEMES['s3']
         s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+        s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN']
         s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
         s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']
         s3store.AWS_USE_SSL = settings['AWS_USE_SSL']
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index da0b2c786..389808306 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -244,7 +244,8 @@ class S3FeedStorageTest(unittest.TestCase):
     def test_parse_credentials(self):
         skip_if_no_boto()
         aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',
-                           'AWS_SECRET_ACCESS_KEY': 'settings_secret'}
+                           'AWS_SECRET_ACCESS_KEY': 'settings_secret',
+                           'AWS_SESSION_TOKEN': 'settings_token'}
         crawler = get_crawler(settings_dict=aws_credentials)
         # Instantiate with crawler
         storage = S3FeedStorage.from_crawler(
@@ -253,12 +254,15 @@ class S3FeedStorageTest(unittest.TestCase):
         )
         self.assertEqual(storage.access_key, 'settings_key')
         self.assertEqual(storage.secret_key, 'settings_secret')
+        self.assertEqual(storage.session_token, 'settings_token')
         # Instantiate directly
         storage = S3FeedStorage('s3://mybucket/export.csv',
                                 aws_credentials['AWS_ACCESS_KEY_ID'],
-                                aws_credentials['AWS_SECRET_ACCESS_KEY'])
+                                aws_credentials['AWS_SECRET_ACCESS_KEY'],
+                                session_token=aws_credentials['AWS_SESSION_TOKEN'])
         self.assertEqual(storage.access_key, 'settings_key')
         self.assertEqual(storage.secret_key, 'settings_secret')
+        self.assertEqual(storage.session_token, 'settings_token')
         # URI priority > settings priority
         storage = S3FeedStorage('s3://uri_key:uri_secret@mybucket/export.csv',
                                 aws_credentials['AWS_ACCESS_KEY_ID'],
@@ -1957,8 +1961,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
 
 class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
 
-    def __init__(self, uri, access_key, secret_key, acl, endpoint_url):
-        super().__init__(uri, access_key, secret_key, acl, endpoint_url)
+    def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs):
+        super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs)
 
 
 class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):
diff --git a/tox.ini b/tox.ini
index 4c4bbff6e..96050223b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,6 +23,7 @@ passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY
+    AWS_SESSION_TOKEN
     GCS_TEST_FILE_URI
     GCS_PROJECT_ID
 #allow tox virtualenv to upgrade pip/wheel/setuptools

From 8e7b96d8a2a6d712be29773b4c69e190d0c1ac7b Mon Sep 17 00:00:00 2001
From: laggardkernel <laggardkernel@users.noreply.github.com>
Date: Tue, 27 Jul 2021 19:29:25 +0800
Subject: [PATCH 421/716] Tweak doc for setting AWS_SESSION_TOKEN
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/feed-exports.rst | 4 +++-
 docs/topics/settings.rst     | 9 ++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index af60de716..2b3217d62 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -200,7 +200,9 @@ passed through the following settings:
 
 -   :setting:`AWS_ACCESS_KEY_ID`
 -   :setting:`AWS_SECRET_ACCESS_KEY`
--   :setting:`AWS_SESSION_TOKEN` (Optional)
+-   :setting:`AWS_SESSION_TOKEN` (only needed for `temporary security credentials`_)
+
+.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
 
 You can also define a custom ACL and custom endpoint for exported feeds using this setting:
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 58daafa6f..1a1a833df 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -209,14 +209,13 @@ such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
 AWS_SESSION_TOKEN
 -----------------
 
-Default: ``None`` (Optional)
+Default: ``None``
 
 The AWS security token used by code that requires access to `Amazon Web services`_,
-such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
+such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
+`temporary security credentials`_.
 
-The security token is only required by a *temporary security credentials*.
-Using of temporary security credentials is discouraged cause the credentials
-are short term. It may expires before the scraping is done.
+.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
 
 .. setting:: AWS_ENDPOINT_URL
 

From d55b6fcad6a792c16022324c6dd6402f8fde8641 Mon Sep 17 00:00:00 2001
From: Aaron Tan <aaronthj82@gmail.com>
Date: Wed, 28 Jul 2021 12:10:34 +1000
Subject: [PATCH 422/716] Fix for duplicate object description error

---
 docs/topics/extensions.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 3cabcefdd..08272a25d 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -328,7 +328,7 @@ Emails can be sent using the MailSender class
 .. module:: scrapy.mail
    :synopsis: MailSender class
 
-.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
+.. class:: MailSender
 
 To see a full list of parameters, including examples on how to instantiate MailSender and using mail settings, see :ref:`topics-email`
 

From 494e0ad8ffc34e7d8079db6dd24fdc8265e81800 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Wed, 28 Jul 2021 14:29:50 -0300
Subject: [PATCH 423/716] Update docs/topics/dynamic-content.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/dynamic-content.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 56c8b6ae9..9706f43fe 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -272,10 +272,10 @@ The following is a simple snippet to illustrate its usage within Scrapy::
 For this example to work, Scrapy needs to be running on top of the
 :ref:`asyncio reactor <install-asyncio>`.
 
-Keep in mind that this is just a proof of concept, since it circumvents
-most of the Scrapy components (middlewares, dupefilter, etc).
-
-The following is a list of 3rd party projects which provide better integration:
+Using pypeteer_ directly circumvents most of the 
+Scrapy components (middlewares, dupefilter, etc). Use
+one of the following Scrapy plugins for better integration
+with Scrapy:
 
 * https://github.com/elacuesta/scrapy-pyppeteer
 * https://github.com/lopuhin/scrapy-pyppeteer

From 0e3d50dd186666362ab8358c9f89036917242c81 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Wed, 28 Jul 2021 14:30:16 -0300
Subject: [PATCH 424/716] Update docs/topics/dynamic-content.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/dynamic-content.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 9706f43fe..e918bc006 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -252,6 +252,7 @@ Since version 2.0, it is possible to integrate libraries that use the
 ``async/await`` syntax. One such library is `pyppeteer`_ (an unnoficial
 Python port of `puppeteer`_), which uses headless Chrome to download and
 render pages.
+
 The following is a simple snippet to illustrate its usage within Scrapy::
 
     import pyppeteer

From 4b62ac6c3ace093d16cf97e737689ac7942d52f9 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 28 Jul 2021 15:00:24 -0300
Subject: [PATCH 425/716] Update headless browser docs to mention playwright

---
 docs/topics/dynamic-content.rst | 55 ++++++++++++++-------------------
 1 file changed, 23 insertions(+), 32 deletions(-)

diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index f96be0bbc..ea5d06210 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -246,55 +246,46 @@ Using a headless browser
 ========================
 
 A `headless browser`_ is a special web browser that provides an API for
-automation.
+automation. By installing the :ref:`asyncio reactor <install-asyncio>`,
+it is possible to integrate ``asyncio``-based libraries which handle headless browsers.
 
-Since version 2.0, it is possible to integrate libraries that use the
-``async/await`` syntax. One such library is `pyppeteer`_ (an unnoficial
-Python port of `puppeteer`_), which uses headless Chrome to download and
-render pages.
+One such library is `playwright-python`_ (an official Python port of `playwright`_).
+The following is a simple snippet to illustrate its usage within a Scrapy spider::
 
-The following is a simple snippet to illustrate its usage within Scrapy::
-
-    import pyppeteer
     import scrapy
+    from playwright.async_api import async_playwright
 
-    class PyppeteerSpider(scrapy.Spider):
-        name = "pyppeteer"
-        start_urls = ["data:,"]  # avoid making an actual upstream request
+    class PlaywrightSpider(scrapy.Spider):
+        name = "playwright"
+        start_urls = ["data:,"]  # avoid using the default Scrapy downloader
 
         async def parse(self, response):
-            browser = await pyppeteer.launch()
-            page = await browser.newPage()
-            await page.goto("https:/example.org")
-            title = await page.title()
-            await page.close()
-            yield {"title": title}
+            async with async_playwright() as pw:
+                browser = await pw.chromium.launch()
+                page = await browser.new_page()
+                await page.goto("https:/example.org")
+                title = await page.title()
+                return {"title": title}
 
-For this example to work, Scrapy needs to be running on top of the
-:ref:`asyncio reactor <install-asyncio>`.
-
-Using pypeteer_ directly circumvents most of the 
-Scrapy components (middlewares, dupefilter, etc). Use
-one of the following Scrapy plugins for better integration
-with Scrapy:
-
-* https://github.com/elacuesta/scrapy-pyppeteer
-* https://github.com/lopuhin/scrapy-pyppeteer
-* https://github.com/clemfromspace/scrapy-puppeteer
 
+However, using `playwright-python`_ directly as in the above example
+circumvents most of the Scrapy components (middlewares, dupefilter, etc).
+We recommend using `scrapy-playwright`_ for a better integration.
 
 .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
-.. _chompjs: https://github.com/Nykakin/chompjs
 .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
+.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
+.. _Splash: https://github.com/scrapinghub/splash
+.. _chompjs: https://github.com/Nykakin/chompjs
 .. _curl: https://curl.haxx.se/
 .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
-.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
 .. _js2xml: https://github.com/scrapinghub/js2xml
-.. _puppeteer: https://pptr.dev/
+.. _playwright-python: https://github.com/microsoft/playwright-python
+.. _playwright: https://github.com/microsoft/playwright
 .. _pyppeteer: https://pyppeteer.github.io/pyppeteer/
 .. _pytesseract: https://github.com/madmaze/pytesseract
+.. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright
 .. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
-.. _Splash: https://github.com/scrapinghub/splash
 .. _tabula-py: https://github.com/chezou/tabula-py
 .. _wget: https://www.gnu.org/software/wget/
 .. _wgrep: https://github.com/stav/wgrep

From cc89f6be381d72a2528c8a672158671305019324 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Thu, 29 Jul 2021 17:12:44 -0300
Subject: [PATCH 426/716] Response.attributes (#5218)

---
 docs/topics/request-response.rst |  7 ++--
 scrapy/http/response/__init__.py | 23 +++++++++----
 scrapy/http/response/text.py     |  8 ++---
 tests/test_http_response.py      | 59 ++++++++++++++++++++++++++++++++
 4 files changed, 82 insertions(+), 15 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index a6a3daf31..d3e08efd4 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -670,9 +670,6 @@ Response objects
 
 .. autoclass:: Response
 
-    A :class:`Response` object represents an HTTP response, which is usually
-    downloaded (by the Downloader) and fed to the Spiders for processing.
-
     :param url: the URL of this response
     :type url: str
 
@@ -829,6 +826,8 @@ Response objects
         handlers, i.e. for ``http(s)`` responses. For other handlers,
         :attr:`protocol` is always ``None``.
 
+    .. autoattribute:: Response.attributes
+
     .. method:: Response.copy()
 
        Returns a new Response which is a copy of this Response.
@@ -925,6 +924,8 @@ TextResponse objects
         A :class:`~scrapy.Selector` instance using the response as
         target. The selector is lazily instantiated on first access.
 
+    .. autoattribute:: TextResponse.attributes
+
     :class:`TextResponse` objects support the following methods in addition to
     the standard :class:`Response` ones:
 
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 185a9bb67..4de6c9b5b 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -4,7 +4,7 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
-from typing import Generator
+from typing import Generator, Tuple
 from urllib.parse import urljoin
 
 from scrapy.exceptions import NotSupported
@@ -16,6 +16,19 @@ from scrapy.utils.trackref import object_ref
 
 
 class Response(object_ref):
+    """An object that represents an HTTP response, which is usually
+    downloaded (by the Downloader) and fed to the Spiders for processing.
+    """
+
+    attributes: Tuple[str, ...] = (
+        "url", "status", "headers", "body", "flags", "request", "certificate", "ip_address", "protocol",
+    )
+    """A tuple of :class:`str` objects containing the name of all public
+    attributes of the class that are also keyword parameters of the
+    ``__init__`` method.
+
+    Currently used by :meth:`Response.replace`.
+    """
 
     def __init__(
         self,
@@ -97,12 +110,8 @@ class Response(object_ref):
         return self.replace()
 
     def replace(self, *args, **kwargs):
-        """Create a new Response with the same attributes except for those
-        given new values.
-        """
-        for x in [
-            "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol",
-        ]:
+        """Create a new Response with the same attributes except for those given new values"""
+        for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index e36e14880..27bd55c07 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
 import json
 import warnings
 from contextlib import suppress
-from typing import Generator
+from typing import Generator, Tuple
 from urllib.parse import urljoin
 
 import parsel
@@ -30,6 +30,8 @@ class TextResponse(Response):
     _DEFAULT_ENCODING = 'ascii'
     _cached_decoded_json = _NONE
 
+    attributes: Tuple[str, ...] = Response.attributes + ("encoding",)
+
     def __init__(self, *args, **kwargs):
         self._encoding = kwargs.pop('encoding', None)
         self._cached_benc = None
@@ -53,10 +55,6 @@ class TextResponse(Response):
         else:
             super()._set_body(body)
 
-    def replace(self, *args, **kwargs):
-        kwargs.setdefault('encoding', self.encoding)
-        return Response.replace(self, *args, **kwargs)
-
     @property
     def encoding(self):
         return self._declared_encoding() or self._body_inferred_encoding()
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 04a594d03..cf34a9e5c 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -820,3 +820,62 @@ class XmlResponseTest(TextResponseTest):
             response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(),
             response.selector.xpath("//s2:elem/text()").getall(),
         )
+
+
+class CustomResponse(TextResponse):
+    attributes = TextResponse.attributes + ("foo", "bar")
+
+    def __init__(self, *args, **kwargs) -> None:
+        self.foo = kwargs.pop("foo", None)
+        self.bar = kwargs.pop("bar", None)
+        self.lost = kwargs.pop("lost", None)
+        super().__init__(*args, **kwargs)
+
+
+class CustomResponseTest(TextResponseTest):
+    response_class = CustomResponse
+
+    def test_copy(self):
+        super().test_copy()
+        r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost")
+        r2 = r1.copy()
+        self.assertIsInstance(r2, self.response_class)
+        self.assertEqual(r1.foo, r2.foo)
+        self.assertEqual(r1.bar, r2.bar)
+        self.assertEqual(r1.lost, "lost")
+        self.assertIsNone(r2.lost)
+
+    def test_replace(self):
+        super().test_replace()
+        r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost")
+
+        r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost")
+        self.assertIsInstance(r2, self.response_class)
+        self.assertEqual(r1.foo, "foo")
+        self.assertEqual(r1.bar, "bar")
+        self.assertEqual(r1.lost, "lost")
+        self.assertEqual(r2.foo, "new-foo")
+        self.assertEqual(r2.bar, "new-bar")
+        self.assertEqual(r2.lost, "new-lost")
+
+        r3 = r1.replace(foo="new-foo", bar="new-bar")
+        self.assertIsInstance(r3, self.response_class)
+        self.assertEqual(r1.foo, "foo")
+        self.assertEqual(r1.bar, "bar")
+        self.assertEqual(r1.lost, "lost")
+        self.assertEqual(r3.foo, "new-foo")
+        self.assertEqual(r3.bar, "new-bar")
+        self.assertIsNone(r3.lost)
+
+        r4 = r1.replace(foo="new-foo")
+        self.assertIsInstance(r4, self.response_class)
+        self.assertEqual(r1.foo, "foo")
+        self.assertEqual(r1.bar, "bar")
+        self.assertEqual(r1.lost, "lost")
+        self.assertEqual(r4.foo, "new-foo")
+        self.assertEqual(r4.bar, "bar")
+        self.assertIsNone(r4.lost)
+
+        with self.assertRaises(TypeError) as ctx:
+            r1.replace(unknown="unknown")
+        self.assertEqual(str(ctx.exception), "__init__() got an unexpected keyword argument 'unknown'")

From 880a4d9493338516aa6d12d602dadf9be60d3053 Mon Sep 17 00:00:00 2001
From: Aaron Tan <70739609+aaron-tan@users.noreply.github.com>
Date: Sun, 1 Aug 2021 11:02:27 +1000
Subject: [PATCH 427/716] Update docs/topics/extensions.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/extensions.rst | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 08272a25d..297e1fdc5 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -323,14 +323,10 @@ domain has finished scraping, including the Scrapy stats collected. The email
 will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS`
 setting.
 
-Emails can be sent using the MailSender class
-
-.. module:: scrapy.mail
-   :synopsis: MailSender class
-
-.. class:: MailSender
-
-To see a full list of parameters, including examples on how to instantiate MailSender and using mail settings, see :ref:`topics-email`
+Emails can be sent using the :class:`~scrapy.mail.MailSender` class. To see a
+full list of parameters, including examples on how to instantiate
+:class:`~scrapy.mail.MailSender` and use mail settings, see
+:ref:`topics-email`.
 
 .. module:: scrapy.extensions.debug
    :synopsis: Extensions for debugging Scrapy

From 2bf2f9d6db89968bcb5df6bc7d093fdd53de5ba5 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Aug 2021 19:44:11 +0500
Subject: [PATCH 428/716] Add Python 3.10b4 tests on Ubuntu.

---
 .github/workflows/tests-ubuntu.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 57188bd63..57c994158 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -45,6 +45,14 @@ jobs:
           env:
             TOXENV: asyncio
 
+        # 3.10-pre
+        - python-version: "3.10.0-beta.4"
+          env:
+            TOXENV: py
+        - python-version: "3.10.0-beta.4"
+          env:
+            TOXENV: asyncio
+
     steps:
     - uses: actions/checkout@v2
 
@@ -54,7 +62,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install system libraries
-      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')
+      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4'
       run: |
         sudo apt-get update
         # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version

From ef6fb933b568c497ab3745284bc2a02725bced1c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Jul 2021 12:02:15 +0500
Subject: [PATCH 429/716] Fix a Python 3.10 logging issue.

---
 scrapy/utils/signal.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index 62808f3ce..fbafc9d45 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -1,5 +1,5 @@
 """Helper functions for working with signals"""
-import collections
+import collections.abc
 import logging
 
 from twisted.internet.defer import DeferredList, Deferred
@@ -21,7 +21,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     Failures instead of exceptions.
     """
     dont_log = named.pop('dont_log', ())
-    dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,)
+    dont_log = tuple(dont_log) if isinstance(dont_log, collections.abc.Sequence) else (dont_log,)
     dont_log += (StopDownload, )
     spider = named.get('spider', None)
     responses = []

From 93bf1ae7e3966d56539411c59d172c2971ee61e0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Aug 2021 20:16:29 +0500
Subject: [PATCH 430/716] Fix tests for the 3.10 TypeError message change.

---
 tests/test_http_response.py     |  2 +-
 tests/test_request_cb_kwargs.py | 14 ++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index cf34a9e5c..c376a46cd 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -878,4 +878,4 @@ class CustomResponseTest(TextResponseTest):
 
         with self.assertRaises(TypeError) as ctx:
             r1.replace(unknown="unknown")
-        self.assertEqual(str(ctx.exception), "__init__() got an unexpected keyword argument 'unknown'")
+        self.assertTrue(str(ctx.exception).endswith("__init__() got an unexpected keyword argument 'unknown'"))
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 145a4e9b2..b68184b87 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -158,12 +158,14 @@ class CallbackKeywordArgumentsTestCase(TestCase):
                 if key in line.getMessage():
                     exceptions[key] = line
         self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
-        self.assertEqual(
-            str(exceptions['takes_less'].exc_info[1]),
-            "parse_takes_less() got an unexpected keyword argument 'number'"
+        self.assertTrue(
+            str(exceptions['takes_less'].exc_info[1]).endswith(
+                "parse_takes_less() got an unexpected keyword argument 'number'"
+            )
         )
         self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
-        self.assertEqual(
-            str(exceptions['takes_more'].exc_info[1]),
-            "parse_takes_more() missing 1 required positional argument: 'other'"
+        self.assertTrue(
+            str(exceptions['takes_more'].exc_info[1]).endswith(
+                "parse_takes_more() missing 1 required positional argument: 'other'"
+            )
         )

From 94baa4b27273e5a779bb977cea4c8eb5301fc3bf Mon Sep 17 00:00:00 2001
From: Mannan2812 <42071936+Mannan2812@users.noreply.github.com>
Date: Fri, 6 Aug 2021 00:53:11 +0530
Subject: [PATCH 431/716] Fix FileFeedStoragePreFeedOptionsTest fails in CI/CD
 pipeline (#5198)

---
 tests/test_feedexport.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 389808306..53e6a2018 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1932,14 +1932,15 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
     maxDiff = None
 
     def test_init(self):
-        settings_dict = {
-            'FEED_URI': 'file:///tmp/foobar',
-            'FEED_STORAGES': {
-                'file': FileFeedStorageWithoutFeedOptions
-            },
-        }
-        crawler = get_crawler(settings_dict=settings_dict)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        with tempfile.NamedTemporaryFile() as temp:
+            settings_dict = {
+                'FEED_URI': f'file:///{temp.name}',
+                'FEED_STORAGES': {
+                    'file': FileFeedStorageWithoutFeedOptions
+                },
+            }
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
         spider = scrapy.Spider("default")
         with warnings.catch_warnings(record=True) as w:
             feed_exporter.open_spider(spider)

From 8e7d2ef13312bfb4ec5e1800f00108806ddc12e8 Mon Sep 17 00:00:00 2001
From: Aaron Tan <aaronthj82@gmail.com>
Date: Sat, 7 Aug 2021 11:44:12 +1000
Subject: [PATCH 432/716] Document JOBDIR option issue #5173

Add JOBDIR setting to the settings page.
Add default JOBDIR setting to global defaults in scrapy.settings.default_settings module.
---
 docs/topics/settings.rst            | 11 +++++++++++
 scrapy/settings/default_settings.py |  2 ++
 2 files changed, 13 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 1a1a833df..5e820b0a9 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -989,6 +989,17 @@ Default: ``{}``
 A dict containing the pipelines enabled by default in Scrapy. You should never
 modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
 
+.. setting:: JOBDIR
+
+JOBDIR
+------
+
+Default: ``''``
+
+A string indicating the directory for storing the required data to keep the state of a single job to enable persistence support. This directory must not be shared by different spiders or jobs/runs of the same spider.
+
+For more info on this setting, see :ref:`topics-jobs`
+
 .. setting:: LOG_ENABLED
 
 LOG_ENABLED
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 4ef330dd2..9137086c0 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -199,6 +199,8 @@ ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 ITEM_PIPELINES = {}
 ITEM_PIPELINES_BASE = {}
 
+JOBDIR = ''
+
 LOG_ENABLED = True
 LOG_ENCODING = 'utf-8'
 LOG_FORMATTER = 'scrapy.logformatter.LogFormatter'

From 48eff4ee8f21535e98baf2bdff91749fee002c10 Mon Sep 17 00:00:00 2001
From: Aaron Tan <aaronthj82@gmail.com>
Date: Sun, 8 Aug 2021 20:52:14 +1000
Subject: [PATCH 433/716] Remove JOBDIR from default settings

---
 scrapy/settings/default_settings.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 9137086c0..4ef330dd2 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -199,8 +199,6 @@ ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 ITEM_PIPELINES = {}
 ITEM_PIPELINES_BASE = {}
 
-JOBDIR = ''
-
 LOG_ENABLED = True
 LOG_ENCODING = 'utf-8'
 LOG_FORMATTER = 'scrapy.logformatter.LogFormatter'

From 954f3035908f7f7f528ce4d3c5245f056645dc7f Mon Sep 17 00:00:00 2001
From: Aaron Tan <70739609+aaron-tan@users.noreply.github.com>
Date: Mon, 9 Aug 2021 22:23:23 +1000
Subject: [PATCH 434/716] Update docs/topics/settings.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/settings.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 5e820b0a9..2ab2020fa 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -996,9 +996,8 @@ JOBDIR
 
 Default: ``''``
 
-A string indicating the directory for storing the required data to keep the state of a single job to enable persistence support. This directory must not be shared by different spiders or jobs/runs of the same spider.
-
-For more info on this setting, see :ref:`topics-jobs`
+A string indicating the directory for storing the state of a crawl when
+:ref:`pausing and resuming crawls <topics-jobs>`.
 
 .. setting:: LOG_ENABLED
 

From 1ba0f68483cfb8aa62759e21b3479ec9bea94beb Mon Sep 17 00:00:00 2001
From: Michel Ace <divtiply@gmail.com>
Date: Tue, 10 Aug 2021 17:09:37 +0200
Subject: [PATCH 435/716] Allow comma-separated values in the rel tag

Comma-separated `rel` values are often seen in the wild, because Google allows it (see https://developers.google.com/search/docs/advanced/guidelines/qualify-outbound-links).
---
 scrapy/utils/misc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 5c986eedc..51cef1e91 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -138,7 +138,7 @@ def md5sum(file):
 
 def rel_has_nofollow(rel):
     """Return True if link rel attribute has nofollow type"""
-    return rel is not None and 'nofollow' in rel.split()
+    return rel is not None and 'nofollow' in rel.replace(',', ' ').split()
 
 
 def create_instance(objcls, settings, crawler, *args, **kwargs):

From 18b6f30a7359d1a798c30888ddd10a1612d8e711 Mon Sep 17 00:00:00 2001
From: Michel Ace <divtiply@gmail.com>
Date: Tue, 10 Aug 2021 21:13:50 +0200
Subject: [PATCH 436/716] Add test for rel_has_nofollow

---
 tests/test_utils_misc/__init__.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index e95a3a316..67367dbfb 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -4,7 +4,7 @@ import unittest
 from unittest import mock
 
 from scrapy.item import Item, Field
-from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules
+from scrapy.utils.misc import arg_to_iter, create_instance, load_object, rel_has_nofollow, set_environ, walk_modules
 
 
 __doctests__ = ['scrapy.utils.misc']
@@ -162,6 +162,12 @@ class UtilsMiscTestCase(unittest.TestCase):
             assert os.environ.get('some_test_environ') == 'test_value'
         assert os.environ.get('some_test_environ') == 'test'
 
+    def test_rel_has_nofollow(self):
+        assert os.environ.get('some_test_environ') is None
+        asert rel_has_nofollow('ugc nofollow') == True
+        asert rel_has_nofollow('ugc,nofollow') == True
+        asert rel_has_nofollow('ugc') == False
+
 
 if __name__ == "__main__":
     unittest.main()

From 07d20a8ce45ab0cbf61d08214db4963302661257 Mon Sep 17 00:00:00 2001
From: Michel Ace <divtiply@gmail.com>
Date: Tue, 10 Aug 2021 21:21:43 +0200
Subject: [PATCH 437/716] Fix test_rel_has_nofollow test

---
 tests/test_utils_misc/__init__.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 67367dbfb..b0d7acd12 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -163,10 +163,9 @@ class UtilsMiscTestCase(unittest.TestCase):
         assert os.environ.get('some_test_environ') == 'test'
 
     def test_rel_has_nofollow(self):
-        assert os.environ.get('some_test_environ') is None
-        asert rel_has_nofollow('ugc nofollow') == True
-        asert rel_has_nofollow('ugc,nofollow') == True
-        asert rel_has_nofollow('ugc') == False
+        assert rel_has_nofollow('ugc nofollow') == True
+        assert rel_has_nofollow('ugc,nofollow') == True
+        assert rel_has_nofollow('ugc') == False
 
 
 if __name__ == "__main__":

From 295f0e2bf5c352c6ddf27a188af10bf122d1c6b0 Mon Sep 17 00:00:00 2001
From: Michel Ace <divtiply@gmail.com>
Date: Tue, 10 Aug 2021 21:38:29 +0200
Subject: [PATCH 438/716] Make flake8 happy

---
 tests/test_utils_misc/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index b0d7acd12..69f593ccd 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -163,9 +163,9 @@ class UtilsMiscTestCase(unittest.TestCase):
         assert os.environ.get('some_test_environ') == 'test'
 
     def test_rel_has_nofollow(self):
-        assert rel_has_nofollow('ugc nofollow') == True
-        assert rel_has_nofollow('ugc,nofollow') == True
-        assert rel_has_nofollow('ugc') == False
+        assert rel_has_nofollow('ugc nofollow') is True
+        assert rel_has_nofollow('ugc,nofollow') is True
+        assert rel_has_nofollow('ugc') is False
 
 
 if __name__ == "__main__":

From ce9d6c658b21a5d9d9605a2683b7a143f2077dfa Mon Sep 17 00:00:00 2001
From: Michel Ace <divtiply@gmail.com>
Date: Tue, 10 Aug 2021 22:21:51 +0200
Subject: [PATCH 439/716] Add more rel_has_nofollow tests

---
 tests/test_utils_misc/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 69f593ccd..47d73a2dd 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -166,6 +166,10 @@ class UtilsMiscTestCase(unittest.TestCase):
         assert rel_has_nofollow('ugc nofollow') is True
         assert rel_has_nofollow('ugc,nofollow') is True
         assert rel_has_nofollow('ugc') is False
+        assert rel_has_nofollow('nofollow') is True
+        assert rel_has_nofollow('nofollowfoo') is False
+        assert rel_has_nofollow('foonofollow') is False
+        assert rel_has_nofollow('ugc,  ,  nofollow') is True
 
 
 if __name__ == "__main__":

From 983b89ad4f72730c37b32c9240a697a4c1f24183 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 11 Aug 2021 10:39:23 +0500
Subject: [PATCH 440/716] Fix SpiderLoaderTest on Python 3.10.

---
 tests/test_spiderloader/__init__.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 4929f1e3e..8a35e9fd7 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -118,6 +118,11 @@ class SpiderLoaderTest(unittest.TestCase):
             settings = Settings({'SPIDER_MODULES': [module],
                                  'SPIDER_LOADER_WARN_ONLY': True})
             spider_loader = SpiderLoader.from_settings(settings)
+            if str(w[0].message).startswith("_SixMetaPathImporter"):
+                # needed on 3.10 because of https://github.com/benjaminp/six/issues/349,
+                # at least until all six versions we can import (including botocore.vendored.six)
+                # are updated to 1.16.0+
+                w.pop(0)
             self.assertIn("Could not load spiders from module", str(w[0].message))
 
             spiders = spider_loader.list()

From 74cee38a4e07c31a8dd2a8772ff18d1b9adf8a6b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 11 Aug 2021 14:19:08 +0500
Subject: [PATCH 441/716] Don't run the asyncio tests on 3.9.

---
 .github/workflows/tests-ubuntu.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 57c994158..81beda5da 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -41,9 +41,6 @@ jobs:
           env:
             TOXENV: extra-deps
             TOX_PIP_VERSION: 20.3.3
-        - python-version: 3.9
-          env:
-            TOXENV: asyncio
 
         # 3.10-pre
         - python-version: "3.10.0-beta.4"

From b63369c148b9a1f87a974e21c5e307a62783d6fd Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 11 Aug 2021 20:02:45 +0500
Subject: [PATCH 442/716] Rename tests/requirements-py3.txt to
 tests/requirements.txt.

---
 tests/{requirements-py3.txt => requirements.txt} | 0
 tox.ini                                          | 4 ++--
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename tests/{requirements-py3.txt => requirements.txt} (100%)

diff --git a/tests/requirements-py3.txt b/tests/requirements.txt
similarity index 100%
rename from tests/requirements-py3.txt
rename to tests/requirements.txt
diff --git a/tox.ini b/tox.ini
index 96050223b..e274fc8d2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,7 +9,7 @@ minversion = 1.7.0
 
 [testenv]
 deps =
-    -rtests/requirements-py3.txt
+    -rtests/requirements.txt
     # mitmproxy does not support PyPy
     # mitmproxy does not support Windows when running Python < 3.7
     # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe
@@ -82,7 +82,7 @@ deps =
     Twisted[http2]==17.9.0
     w3lib==1.17.0
     zope.interface==4.1.3
-    -rtests/requirements-py3.txt
+    -rtests/requirements.txt
 
     # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
     # above, hence we do not install it in pinned environments at the moment

From 2814e0e1972fa38151b6800c881d49f50edf9c6b Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Mon, 16 Aug 2021 16:22:01 +0500
Subject: [PATCH 443/716] Disable builtin middlewares in spider middleware
 tests. (#5229)

---
 tests/test_spidermiddleware.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 78e926adc..b39576996 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -15,7 +15,7 @@ class SpiderMiddlewareTestCase(TestCase):
     def setUp(self):
         self.request = Request('http://example.com/index.html')
         self.response = Response(self.request.url, request=self.request)
-        self.crawler = get_crawler(Spider)
+        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}})
         self.spider = self.crawler._create_spider('foo')
         self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
 

From 8bbaea9892003769672204a8ff4e989b5aab5ceb Mon Sep 17 00:00:00 2001
From: databender <umairnsr87@gmail.com>
Date: Mon, 16 Aug 2021 16:57:43 +0530
Subject: [PATCH 444/716] updated documentation for python version for reppy

---
 docs/topics/downloader-middleware.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 80c6c2c37..222dda685 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1073,6 +1073,8 @@ In order to use this parser:
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``
 
+* only works with python 3.8 and earlier
+
 .. _rerp-parser:
 
 Robotexclusionrulesparser

From 1a8b98843aee548a52faa36f5360a81a1624e208 Mon Sep 17 00:00:00 2001
From: databender <umairnsr87@gmail.com>
Date: Mon, 16 Aug 2021 17:00:05 +0530
Subject: [PATCH 445/716] updated documentation for python version for reppy

---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 222dda685..2c00ad45d 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1073,7 +1073,7 @@ In order to use this parser:
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``
 
-* only works with python 3.8 and earlier
+* Only works with python 3.8 and earlier
 
 .. _rerp-parser:
 

From cc1cb2de0c6e91393ceb6872c174aa2ac06c07ac Mon Sep 17 00:00:00 2001
From: databender <umairnsr87@gmail.com>
Date: Mon, 16 Aug 2021 17:21:47 +0530
Subject: [PATCH 446/716] updated suggested changes

---
 docs/topics/downloader-middleware.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 2c00ad45d..fa211fb75 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1068,12 +1068,13 @@ Native implementation, provides better speed than Protego.
 
 In order to use this parser:
 
+.. warning:: Does not support Python 3.9+
+
 * Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
 
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``
 
-* Only works with python 3.8 and earlier
 
 .. _rerp-parser:
 

From 013ac90f6129a9e8b862d66ca2ffa8f0f1fd674e Mon Sep 17 00:00:00 2001
From: umair ansari <umairnsr87@gmail.com>
Date: Mon, 16 Aug 2021 18:00:06 +0530
Subject: [PATCH 447/716] Update docs/topics/downloader-middleware.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/downloader-middleware.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index fa211fb75..8323bc564 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1068,10 +1068,10 @@ Native implementation, provides better speed than Protego.
 
 In order to use this parser:
 
-.. warning:: Does not support Python 3.9+
-
 * Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
 
+  .. warning:: Does not support Python 3.9+
+
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``
 

From ebddb77a331c6290e64e449ef9847e33b323a146 Mon Sep 17 00:00:00 2001
From: databender <umairnsr87@gmail.com>
Date: Mon, 16 Aug 2021 18:08:26 +0530
Subject: [PATCH 448/716] updated suggested changes after review

---
 docs/topics/downloader-middleware.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index fa211fb75..4d7c87404 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1068,10 +1068,10 @@ Native implementation, provides better speed than Protego.
 
 In order to use this parser:
 
-.. warning:: Does not support Python 3.9+
-
 * Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
 
+.. warning:: Does not support Python 3.9+
+
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``
 

From bcf38a67194f25db66334af18bdf49b34c6a0c39 Mon Sep 17 00:00:00 2001
From: databender <umairnsr87@gmail.com>
Date: Wed, 18 Aug 2021 14:48:47 +0530
Subject: [PATCH 449/716] added upstream issue for not supported python version

---
 docs/topics/downloader-middleware.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 4d7c87404..089d5683a 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1070,7 +1070,9 @@ In order to use this parser:
 
 * Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
 
-.. warning:: Does not support Python 3.9+
+ .. warning:: `Upstream issue #122
+      <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in
+      Python 3.9+.
 
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``

From d623ed15d1a79a91b55aa7aae2d942aac94abfdf Mon Sep 17 00:00:00 2001
From: databender <umairnsr87@gmail.com>
Date: Wed, 18 Aug 2021 14:51:03 +0530
Subject: [PATCH 450/716] indentation updated

---
 docs/topics/downloader-middleware.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 089d5683a..928a59bf1 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1070,9 +1070,9 @@ In order to use this parser:
 
 * Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
 
- .. warning:: `Upstream issue #122
-      <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in
-      Python 3.9+.
+.. warning:: `Upstream issue #122
+  <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in
+  Python 3.9+.
 
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``

From 2d2581c68f35799dc4372a257eaa8dbb5208481d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 18 Aug 2021 12:46:42 +0200
Subject: [PATCH 451/716] Move documentation about avoiding bans into a topic
 of its own (#4039)

---
 docs/index.rst                |   4 +
 docs/topics/avoiding-bans.rst | 340 ++++++++++++++++++++++++++++++++++
 docs/topics/practices.rst     |  34 ----
 3 files changed, 344 insertions(+), 34 deletions(-)
 create mode 100644 docs/topics/avoiding-bans.rst

diff --git a/docs/index.rst b/docs/index.rst
index 433798aa8..7647b3781 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -155,6 +155,7 @@ Solving specific problems
    topics/debug
    topics/contracts
    topics/practices
+   topics/avoiding-bans
    topics/broad-crawls
    topics/developer-tools
    topics/dynamic-content
@@ -179,6 +180,9 @@ Solving specific problems
 :doc:`topics/practices`
     Get familiar with some Scrapy common practices.
 
+:doc:`topics/avoiding-bans`
+    Avoid getting banned from websites.
+
 :doc:`topics/broad-crawls`
     Tune Scrapy for crawling a lot domains in parallel.
 
diff --git a/docs/topics/avoiding-bans.rst b/docs/topics/avoiding-bans.rst
new file mode 100644
index 000000000..59f0da191
--- /dev/null
+++ b/docs/topics/avoiding-bans.rst
@@ -0,0 +1,340 @@
+.. _bans:
+
+=============
+Avoiding bans
+=============
+
+This topic covers some of the strategies that you can follow to avoid getting
+different or bad responses from a website that you are crawling due to filters
+such as regional filters, web browser filters, etc.
+
+.. _avoiding-crawls:
+
+Avoiding crawls
+===============
+
+The best way not to be banned from a website is not to send requests to it in
+the first place.
+
+One way to avoid crawling a website is to find the desired dataset through
+other means. For example, you can use Google’s `dataset search engine`_.
+
+If the target website is the only or best source of the desired information,
+and you only need to extract the data on a monthly basis or a lower frequency,
+you may be able to crawl a public snapshot of the target website instead.
+`Common Crawl`_ is an open repository of web crawl data that you can access
+freely. It contains monthly snapshots of a wide variety of websites and, if you
+are lucky, your target website will be among them.
+
+.. _Common Crawl: https://commoncrawl.org/
+.. _dataset search engine: https://datasetsearch.research.google.com/
+
+
+.. _being-polite:
+
+Being polite
+============
+
+To avoid being banned, you should first avoid giving a website reasons to ban
+you.
+
+.. _identifying-yourself:
+
+Identifying yourself
+--------------------
+
+If your crawling has a noticeable negative impact on a website or you crawl
+content that should not be crawled, website administrators will need to do
+something.
+
+Set :setting:`USER_AGENT` to a value that uniquely identifies your spider and
+includes contact information, so that website administrators can contact you.
+
+
+.. _following-robotstxt:
+
+Following robots.txt guidelines
+-------------------------------
+
+Some websites provide a ``robots.txt`` file at their root path (e.g.
+``http://example.com/robots.txt``) that describes the guidelines that they wish
+bots to follow when crawling their website.
+
+Before you start writing a spider for a website, read their ``robots.txt``
+file and implement your spider following its guidelines. See the `robots.txt
+standard draft`_ or the `robots.txt Google specification`_ for information on
+how to read ``robots.txt`` files.
+
+To ensure that your spider does not crawl pages restricted by ``robots.txt``
+guidelines, set :setting:`ROBOTSTXT_OBEY` to ``True`` to enable the
+:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
+middleware. When you do, if your spider attempts to crawl a restricted page,
+this middleware aborts that request with the following message::
+
+    Forbidden by robots.txt
+
+Also set :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
+:setting:`DOWNLOAD_DELAY` to values that comply with the ``Crawl-Delay`` or
+``Request-Rate`` directives from the ``robots.txt`` guidelines.
+
+You may also use the :ref:`AutoThrottle extension <topics-autothrottle>` on top
+of that, so that when the target website experiences a high load, your spider
+automatically switches to higher download delays.
+
+.. _robots.txt Google specification: https://developers.google.com/search/reference/robots_txt
+.. _robots.txt standard draft: https://tools.ietf.org/html/draft-koster-rep-00
+
+
+.. _choosing-crawl-speed:
+
+Finding the right guidelines on your own
+----------------------------------------
+
+If a website does not specify a desired download delay, or does not provide a
+``robots.txt`` file, you should make an effort to find out the right values for
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY` that
+will not have a noticeable negative impact on the target website.
+
+Use a service like `SimilarWeb`_ to find out the amount of monthly traffic that
+the target website receives, and choose concurrency and delay values that will
+not cause a noticeable traffic increase.
+
+.. _SimilarWeb: https://www.similarweb.com
+
+
+.. _filters-and-challenges:
+
+Bypassing filters and solving challenges
+========================================
+
+Some websites implement filters and challenges that aim to deny access or alter
+their content based on aspects of the visitor, such as the country where they
+are or the web browsing tool they use.
+
+.. _regional-filter:
+
+Bypassing regional filters
+--------------------------
+
+Some websites send different or bad responses based on the region or country
+associated to your `IP address`_.
+
+To bypass these filters, get access to a `proxy server`_ that has an outgoing
+IP address from a region that gets the desired responses.
+
+Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
+middleware to configure your spider to use that proxy.
+
+.. _IP address: https://en.wikipedia.org/wiki/IP_address
+.. _proxy server: https://en.wikipedia.org/wiki/Proxy_server
+
+
+.. _web-browser-filter:
+
+Bypassing web browser filters
+-----------------------------
+
+Some websites send different or bad responses if they detect that your request
+does not come from a web browser.
+
+To bypass these filters, switch your :setting:`USER_AGENT` to a value copied
+from those that popular web browsers use. In some rare cases, you may need a
+user agent string from a specific web browser.
+
+There are multiple Scrapy plugins that can rotate your requests through popular
+web browser user agent strings, such as scrapy-fake-useragent_,
+scrapy-random-useragent_ or Scrapy-UserAgents_.
+
+For advanced web browser filters,
+:ref:`pre-rendering JavaScript <topics-javascript-rendering>` or
+:ref:`using a headless browser <topics-headless-browsing>` may be necessary.
+Use these options only as a last resort, however, because they cause a higher
+load per request on the target website.
+
+.. _scrapy-fake-useragent: https://github.com/alecxe/scrapy-fake-useragent
+.. _scrapy-random-useragent: https://github.com/cleocn/scrapy-random-useragent
+.. _Scrapy-UserAgents: https://pypi.org/project/Scrapy-UserAgents/
+
+
+.. _request-delay-filter:
+
+Bypassing request delay filters
+-------------------------------
+
+Some websites may ban your IP after they detect that your requests use a
+constant download delay.
+
+To help bypassing these filters, the :setting:`RANDOMIZE_DOWNLOAD_DELAY`
+setting is enabled by default. When that is not enough, an
+:ref:`IP address rotation solution <ip-rotation>` may be much more effective.
+
+
+.. _isp-filter:
+
+Bypassing internet service provider filters
+-------------------------------------------
+
+Some websites send different or bad responses if they detect that your request
+comes from an IP address that belongs to a `data center`_, as opposed to a
+residential IP address from an `internet service provider`_ or a mobile IP
+address from a `mobile network`_.
+
+To bypass these filters, get access to a proxy server that has an outgoing IP
+address that is either residential or mobile. Note that you may also get
+different responses depending on whether your IP address is residential or
+mobile.
+
+Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
+middleware to configure your spider to use that proxy.
+
+.. _data center: https://en.wikipedia.org/wiki/Data_center
+.. _internet service provider: https://en.wikipedia.org/wiki/Internet_service_provider
+.. _mobile network: https://en.wikipedia.org/wiki/Cellular_network
+
+
+.. _captcha:
+
+Solving CAPTCHA challenges
+--------------------------
+
+Some websites require you to solve a `CAPTCHA challenge`_ to get the desired
+response.
+
+To bypass these filters, several options exist:
+
+-   You could have your spider present the CAPTCHA challenge to you and wait
+    for you to solve it manually.
+
+-   Some CAPTCHA challenges can be solved using an `optical character
+    recognition`_ (OCR) solution such as pytesseract_.
+
+-   Paid CAPTCHA solving services exist.
+
+Whichever solution you choose, implement it as a :ref:`downloader middleware
+<topics-downloader-middleware>` that automatically detects CAPTCHA challenges
+in responses and solves them, so that your spider code only receives successful
+responses.
+
+.. _CAPTCHA challenge: https://en.wikipedia.org/wiki/CAPTCHA
+.. _optical character recognition: https://en.wikipedia.org/wiki/Optical_character_recognition
+.. _pytesseract: https://github.com/madmaze/pytesseract
+
+
+.. _ip-rotation:
+
+IP address rotation solutions
+=============================
+
+See below some of the different solutions there are to have your requests use
+different outgoing IP addresses.
+
+When using this approach, remember to set :setting:`COOKIES_ENABLED` to
+``False`` to disable global cookie handling. This prevents websites from
+identifying two requests as coming from the same user agent even if they come
+from different IP addresses and have different user-agent strings. You can
+still include some cookies manually in your requests. Define them through the
+``Cookies`` header of your requests. See
+:class:`Request.headers <scrapy.http.Request.headers>`.
+
+.. _smart-proxy:
+
+Smart proxies
+-------------
+
+An increasing number of websites use solutions that apply many of the above
+filters and challenges at the same time.
+
+There are paid proxy services, like `Zyte Smart Proxy Manager`_, that
+automatically bypass website filters and challenges, so that your spider only
+gets successful responses. They also allow managing sessions to simulate user
+behavior.
+
+For Zyte Smart Proxy Manager, installing scrapy-crawlera_ will offer advanced
+integration with Scrapy. For other services, use the
+:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` middleware
+or implement your own :ref:`downloader middleware
+<topics-downloader-middleware>`.
+
+.. _scrapy-crawlera: https://scrapy-crawlera.readthedocs.io/en/latest/
+.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
+
+
+.. _rotating-proxy:
+
+Rotating proxies
+----------------
+
+Rotating proxy services like ProxyMesh_ send different requests through
+different proxies. This can decrease the likelihood of being affected by some
+filters or challenges.
+
+.. _ProxyMesh: https://proxymesh.com/
+
+
+.. _free-proxies:
+
+Free proxies
+------------
+
+You can easily find lists of free proxies in the internet, and you can use
+a solution like `scrapy-rotating-proxies`_ to configure multiple proxies in
+your spider and have requests rotate through them automatically.
+
+This approach, however, has serious drawbacks:
+
+-   Free proxies may stop working at any moment. You need to implement a way to
+    refresh your list of free proxies.
+
+-   In addition to handling occasional bad responses from websites, you
+    need to handle all kinds of bad responses from proxies. You may even need
+    to inspect the response body to determine if a response comes from the
+    target website or from a misbehaving proxy.
+
+-   Advanced antibot solutions may automatically detect and filter out traffic
+    from free proxies.
+
+.. _scrapy-rotating-proxies: https://github.com/TeamHG-Memex/scrapy-rotating-proxies
+
+
+.. _custom-rotating-proxy:
+
+Custom rotating proxy
+---------------------
+
+If you have spare servers, you can set them up as proxies and use scrapoxy_ to
+build a custom proxy that rotates traffic through them. However, the initial
+setup can be complex, and your requests will be vulnerable to
+:ref:`internet service provider filtering <isp-filter>`.
+
+.. _scrapoxy: https://scrapoxy.io/
+
+
+.. _tor:
+
+The Tor network
+---------------
+
+It is possible to send requests through the `Tor network`_.
+
+The initial setup to have Scrapy working with Tor is not straightforward.
+Use a search engine to find up-to-date documentation specific to using
+Scrapy and Tor together.
+
+The main drawback of using the Tor network is that traffic can be extremely
+slow.
+
+.. _Tor network: https://en.wikipedia.org/wiki/Tor_(anonymity_network)
+
+
+.. _commercial-support:
+
+Seeking professional help
+=========================
+
+Avoiding bans, filters and challenges can be difficult and tricky, and may
+sometimes require special infrastructure.
+
+If you find yourself unable to prevent your spider from getting bad responses,
+consider contacting `commercial support`_.
+
+.. _commercial support: https://scrapy.org/support/
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 732eba587..a7a6fd129 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -226,39 +226,5 @@ crawl::
     curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2
     curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3
 
-.. _bans:
 
-Avoiding getting banned
-=======================
-
-Some websites implement certain measures to prevent bots from crawling them,
-with varying degrees of sophistication. Getting around those measures can be
-difficult and tricky, and may sometimes require special infrastructure. Please
-consider contacting `commercial support`_ if in doubt.
-
-Here are some tips to keep in mind when dealing with these kinds of sites:
-
-* rotate your user agent from a pool of well-known ones from browsers (google
-  around to get a list of them)
-* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use
-  cookies to spot bot behaviour
-* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting.
-* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites
-  directly
-* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
-  services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
-  super proxy that you can attach your own proxies to.
-* use a highly distributed downloader that circumvents bans internally, so you
-  can just focus on parsing clean pages. One example of such downloaders is
-  `Zyte Smart Proxy Manager`_
-
-If you are still unable to prevent your bot getting banned, consider contacting
-`commercial support`_.
-
-.. _Tor project: https://www.torproject.org/
-.. _commercial support: https://scrapy.org/support/
-.. _ProxyMesh: https://proxymesh.com/
-.. _Google cache: http://www.googleguide.com/cached_pages.html
 .. _testspiders: https://github.com/scrapinghub/testspiders
-.. _scrapoxy: https://scrapoxy.io/
-.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/

From 572d347b3bc0149042f04ea83aff4a4f8fc7a831 Mon Sep 17 00:00:00 2001
From: databender <umairnsr87@gmail.com>
Date: Wed, 18 Aug 2021 16:17:52 +0530
Subject: [PATCH 452/716] warning view updated

---
 docs/topics/downloader-middleware.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 928a59bf1..99d57bda9 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1070,9 +1070,8 @@ In order to use this parser:
 
 * Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
 
-.. warning:: `Upstream issue #122
-  <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in
-  Python 3.9+.
+    .. warning:: `Upstream issue #122
+        <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+.
 
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.ReppyRobotParser``

From cd17c829cf0d7a006ab5594d56fe182a0ffc71d6 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 23 Aug 2021 19:55:35 +0500
Subject: [PATCH 453/716] Revert "Move documentation about avoiding bans into a
 topic of its own (#4039)"

This reverts commit 2d2581c68f35799dc4372a257eaa8dbb5208481d.
---
 docs/index.rst                |   4 -
 docs/topics/avoiding-bans.rst | 340 ----------------------------------
 docs/topics/practices.rst     |  34 ++++
 3 files changed, 34 insertions(+), 344 deletions(-)
 delete mode 100644 docs/topics/avoiding-bans.rst

diff --git a/docs/index.rst b/docs/index.rst
index 7647b3781..433798aa8 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -155,7 +155,6 @@ Solving specific problems
    topics/debug
    topics/contracts
    topics/practices
-   topics/avoiding-bans
    topics/broad-crawls
    topics/developer-tools
    topics/dynamic-content
@@ -180,9 +179,6 @@ Solving specific problems
 :doc:`topics/practices`
     Get familiar with some Scrapy common practices.
 
-:doc:`topics/avoiding-bans`
-    Avoid getting banned from websites.
-
 :doc:`topics/broad-crawls`
     Tune Scrapy for crawling a lot domains in parallel.
 
diff --git a/docs/topics/avoiding-bans.rst b/docs/topics/avoiding-bans.rst
deleted file mode 100644
index 59f0da191..000000000
--- a/docs/topics/avoiding-bans.rst
+++ /dev/null
@@ -1,340 +0,0 @@
-.. _bans:
-
-=============
-Avoiding bans
-=============
-
-This topic covers some of the strategies that you can follow to avoid getting
-different or bad responses from a website that you are crawling due to filters
-such as regional filters, web browser filters, etc.
-
-.. _avoiding-crawls:
-
-Avoiding crawls
-===============
-
-The best way not to be banned from a website is not to send requests to it in
-the first place.
-
-One way to avoid crawling a website is to find the desired dataset through
-other means. For example, you can use Google’s `dataset search engine`_.
-
-If the target website is the only or best source of the desired information,
-and you only need to extract the data on a monthly basis or a lower frequency,
-you may be able to crawl a public snapshot of the target website instead.
-`Common Crawl`_ is an open repository of web crawl data that you can access
-freely. It contains monthly snapshots of a wide variety of websites and, if you
-are lucky, your target website will be among them.
-
-.. _Common Crawl: https://commoncrawl.org/
-.. _dataset search engine: https://datasetsearch.research.google.com/
-
-
-.. _being-polite:
-
-Being polite
-============
-
-To avoid being banned, you should first avoid giving a website reasons to ban
-you.
-
-.. _identifying-yourself:
-
-Identifying yourself
---------------------
-
-If your crawling has a noticeable negative impact on a website or you crawl
-content that should not be crawled, website administrators will need to do
-something.
-
-Set :setting:`USER_AGENT` to a value that uniquely identifies your spider and
-includes contact information, so that website administrators can contact you.
-
-
-.. _following-robotstxt:
-
-Following robots.txt guidelines
--------------------------------
-
-Some websites provide a ``robots.txt`` file at their root path (e.g.
-``http://example.com/robots.txt``) that describes the guidelines that they wish
-bots to follow when crawling their website.
-
-Before you start writing a spider for a website, read their ``robots.txt``
-file and implement your spider following its guidelines. See the `robots.txt
-standard draft`_ or the `robots.txt Google specification`_ for information on
-how to read ``robots.txt`` files.
-
-To ensure that your spider does not crawl pages restricted by ``robots.txt``
-guidelines, set :setting:`ROBOTSTXT_OBEY` to ``True`` to enable the
-:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
-middleware. When you do, if your spider attempts to crawl a restricted page,
-this middleware aborts that request with the following message::
-
-    Forbidden by robots.txt
-
-Also set :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
-:setting:`DOWNLOAD_DELAY` to values that comply with the ``Crawl-Delay`` or
-``Request-Rate`` directives from the ``robots.txt`` guidelines.
-
-You may also use the :ref:`AutoThrottle extension <topics-autothrottle>` on top
-of that, so that when the target website experiences a high load, your spider
-automatically switches to higher download delays.
-
-.. _robots.txt Google specification: https://developers.google.com/search/reference/robots_txt
-.. _robots.txt standard draft: https://tools.ietf.org/html/draft-koster-rep-00
-
-
-.. _choosing-crawl-speed:
-
-Finding the right guidelines on your own
-----------------------------------------
-
-If a website does not specify a desired download delay, or does not provide a
-``robots.txt`` file, you should make an effort to find out the right values for
-:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY` that
-will not have a noticeable negative impact on the target website.
-
-Use a service like `SimilarWeb`_ to find out the amount of monthly traffic that
-the target website receives, and choose concurrency and delay values that will
-not cause a noticeable traffic increase.
-
-.. _SimilarWeb: https://www.similarweb.com
-
-
-.. _filters-and-challenges:
-
-Bypassing filters and solving challenges
-========================================
-
-Some websites implement filters and challenges that aim to deny access or alter
-their content based on aspects of the visitor, such as the country where they
-are or the web browsing tool they use.
-
-.. _regional-filter:
-
-Bypassing regional filters
---------------------------
-
-Some websites send different or bad responses based on the region or country
-associated to your `IP address`_.
-
-To bypass these filters, get access to a `proxy server`_ that has an outgoing
-IP address from a region that gets the desired responses.
-
-Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
-middleware to configure your spider to use that proxy.
-
-.. _IP address: https://en.wikipedia.org/wiki/IP_address
-.. _proxy server: https://en.wikipedia.org/wiki/Proxy_server
-
-
-.. _web-browser-filter:
-
-Bypassing web browser filters
------------------------------
-
-Some websites send different or bad responses if they detect that your request
-does not come from a web browser.
-
-To bypass these filters, switch your :setting:`USER_AGENT` to a value copied
-from those that popular web browsers use. In some rare cases, you may need a
-user agent string from a specific web browser.
-
-There are multiple Scrapy plugins that can rotate your requests through popular
-web browser user agent strings, such as scrapy-fake-useragent_,
-scrapy-random-useragent_ or Scrapy-UserAgents_.
-
-For advanced web browser filters,
-:ref:`pre-rendering JavaScript <topics-javascript-rendering>` or
-:ref:`using a headless browser <topics-headless-browsing>` may be necessary.
-Use these options only as a last resort, however, because they cause a higher
-load per request on the target website.
-
-.. _scrapy-fake-useragent: https://github.com/alecxe/scrapy-fake-useragent
-.. _scrapy-random-useragent: https://github.com/cleocn/scrapy-random-useragent
-.. _Scrapy-UserAgents: https://pypi.org/project/Scrapy-UserAgents/
-
-
-.. _request-delay-filter:
-
-Bypassing request delay filters
--------------------------------
-
-Some websites may ban your IP after they detect that your requests use a
-constant download delay.
-
-To help bypassing these filters, the :setting:`RANDOMIZE_DOWNLOAD_DELAY`
-setting is enabled by default. When that is not enough, an
-:ref:`IP address rotation solution <ip-rotation>` may be much more effective.
-
-
-.. _isp-filter:
-
-Bypassing internet service provider filters
--------------------------------------------
-
-Some websites send different or bad responses if they detect that your request
-comes from an IP address that belongs to a `data center`_, as opposed to a
-residential IP address from an `internet service provider`_ or a mobile IP
-address from a `mobile network`_.
-
-To bypass these filters, get access to a proxy server that has an outgoing IP
-address that is either residential or mobile. Note that you may also get
-different responses depending on whether your IP address is residential or
-mobile.
-
-Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
-middleware to configure your spider to use that proxy.
-
-.. _data center: https://en.wikipedia.org/wiki/Data_center
-.. _internet service provider: https://en.wikipedia.org/wiki/Internet_service_provider
-.. _mobile network: https://en.wikipedia.org/wiki/Cellular_network
-
-
-.. _captcha:
-
-Solving CAPTCHA challenges
---------------------------
-
-Some websites require you to solve a `CAPTCHA challenge`_ to get the desired
-response.
-
-To bypass these filters, several options exist:
-
--   You could have your spider present the CAPTCHA challenge to you and wait
-    for you to solve it manually.
-
--   Some CAPTCHA challenges can be solved using an `optical character
-    recognition`_ (OCR) solution such as pytesseract_.
-
--   Paid CAPTCHA solving services exist.
-
-Whichever solution you choose, implement it as a :ref:`downloader middleware
-<topics-downloader-middleware>` that automatically detects CAPTCHA challenges
-in responses and solves them, so that your spider code only receives successful
-responses.
-
-.. _CAPTCHA challenge: https://en.wikipedia.org/wiki/CAPTCHA
-.. _optical character recognition: https://en.wikipedia.org/wiki/Optical_character_recognition
-.. _pytesseract: https://github.com/madmaze/pytesseract
-
-
-.. _ip-rotation:
-
-IP address rotation solutions
-=============================
-
-See below some of the different solutions there are to have your requests use
-different outgoing IP addresses.
-
-When using this approach, remember to set :setting:`COOKIES_ENABLED` to
-``False`` to disable global cookie handling. This prevents websites from
-identifying two requests as coming from the same user agent even if they come
-from different IP addresses and have different user-agent strings. You can
-still include some cookies manually in your requests. Define them through the
-``Cookies`` header of your requests. See
-:class:`Request.headers <scrapy.http.Request.headers>`.
-
-.. _smart-proxy:
-
-Smart proxies
--------------
-
-An increasing number of websites use solutions that apply many of the above
-filters and challenges at the same time.
-
-There are paid proxy services, like `Zyte Smart Proxy Manager`_, that
-automatically bypass website filters and challenges, so that your spider only
-gets successful responses. They also allow managing sessions to simulate user
-behavior.
-
-For Zyte Smart Proxy Manager, installing scrapy-crawlera_ will offer advanced
-integration with Scrapy. For other services, use the
-:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` middleware
-or implement your own :ref:`downloader middleware
-<topics-downloader-middleware>`.
-
-.. _scrapy-crawlera: https://scrapy-crawlera.readthedocs.io/en/latest/
-.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
-
-
-.. _rotating-proxy:
-
-Rotating proxies
-----------------
-
-Rotating proxy services like ProxyMesh_ send different requests through
-different proxies. This can decrease the likelihood of being affected by some
-filters or challenges.
-
-.. _ProxyMesh: https://proxymesh.com/
-
-
-.. _free-proxies:
-
-Free proxies
-------------
-
-You can easily find lists of free proxies in the internet, and you can use
-a solution like `scrapy-rotating-proxies`_ to configure multiple proxies in
-your spider and have requests rotate through them automatically.
-
-This approach, however, has serious drawbacks:
-
--   Free proxies may stop working at any moment. You need to implement a way to
-    refresh your list of free proxies.
-
--   In addition to handling occasional bad responses from websites, you
-    need to handle all kinds of bad responses from proxies. You may even need
-    to inspect the response body to determine if a response comes from the
-    target website or from a misbehaving proxy.
-
--   Advanced antibot solutions may automatically detect and filter out traffic
-    from free proxies.
-
-.. _scrapy-rotating-proxies: https://github.com/TeamHG-Memex/scrapy-rotating-proxies
-
-
-.. _custom-rotating-proxy:
-
-Custom rotating proxy
----------------------
-
-If you have spare servers, you can set them up as proxies and use scrapoxy_ to
-build a custom proxy that rotates traffic through them. However, the initial
-setup can be complex, and your requests will be vulnerable to
-:ref:`internet service provider filtering <isp-filter>`.
-
-.. _scrapoxy: https://scrapoxy.io/
-
-
-.. _tor:
-
-The Tor network
----------------
-
-It is possible to send requests through the `Tor network`_.
-
-The initial setup to have Scrapy working with Tor is not straightforward.
-Use a search engine to find up-to-date documentation specific to using
-Scrapy and Tor together.
-
-The main drawback of using the Tor network is that traffic can be extremely
-slow.
-
-.. _Tor network: https://en.wikipedia.org/wiki/Tor_(anonymity_network)
-
-
-.. _commercial-support:
-
-Seeking professional help
-=========================
-
-Avoiding bans, filters and challenges can be difficult and tricky, and may
-sometimes require special infrastructure.
-
-If you find yourself unable to prevent your spider from getting bad responses,
-consider contacting `commercial support`_.
-
-.. _commercial support: https://scrapy.org/support/
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index a7a6fd129..732eba587 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -226,5 +226,39 @@ crawl::
     curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2
     curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3
 
+.. _bans:
 
+Avoiding getting banned
+=======================
+
+Some websites implement certain measures to prevent bots from crawling them,
+with varying degrees of sophistication. Getting around those measures can be
+difficult and tricky, and may sometimes require special infrastructure. Please
+consider contacting `commercial support`_ if in doubt.
+
+Here are some tips to keep in mind when dealing with these kinds of sites:
+
+* rotate your user agent from a pool of well-known ones from browsers (google
+  around to get a list of them)
+* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use
+  cookies to spot bot behaviour
+* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting.
+* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites
+  directly
+* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
+  services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
+  super proxy that you can attach your own proxies to.
+* use a highly distributed downloader that circumvents bans internally, so you
+  can just focus on parsing clean pages. One example of such downloaders is
+  `Zyte Smart Proxy Manager`_
+
+If you are still unable to prevent your bot getting banned, consider contacting
+`commercial support`_.
+
+.. _Tor project: https://www.torproject.org/
+.. _commercial support: https://scrapy.org/support/
+.. _ProxyMesh: https://proxymesh.com/
+.. _Google cache: http://www.googleguide.com/cached_pages.html
 .. _testspiders: https://github.com/scrapinghub/testspiders
+.. _scrapoxy: https://scrapoxy.io/
+.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/

From 3f635eb683821667b7a46a99531a95a8c05b6e1b Mon Sep 17 00:00:00 2001
From: "Matsievskiy S.V" <seregaxvm.main@gmail.com>
Date: Tue, 24 Aug 2021 12:05:50 +0300
Subject: [PATCH 454/716] Extract domain from genspider URL (#4439)

---
 scrapy/commands/genspider.py | 12 +++++++++++-
 tests/test_commands.py       | 30 ++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 5f44daa70..2082a4974 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -4,6 +4,7 @@ import string
 
 from importlib import import_module
 from os.path import join, dirname, abspath, exists, splitext
+from urllib.parse import urlparse
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -22,6 +23,14 @@ def sanitize_module_name(module_name):
     return module_name
 
 
+def extract_domain(url):
+    """Extract domain name from URL string"""
+    o = urlparse(url)
+    if o.scheme == '' and o.netloc == '':
+        o = urlparse("//" + url.lstrip("/"))
+    return o.netloc
+
+
 class Command(ScrapyCommand):
 
     requires_project = False
@@ -59,7 +68,8 @@ class Command(ScrapyCommand):
         if len(args) != 2:
             raise UsageError()
 
-        name, domain = args[0:2]
+        name, url = args[0:2]
+        domain = extract_domain(url)
         module = sanitize_module_name(name)
 
         if self.settings.get('BOT_NAME') == module:
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 74b917d93..086286b3a 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -3,6 +3,7 @@ import json
 import optparse
 import os
 import platform
+import re
 import subprocess
 import sys
 import tempfile
@@ -94,6 +95,15 @@ class ProjectTest(unittest.TestCase):
 
         return p, to_unicode(stdout), to_unicode(stderr)
 
+    def find_in_file(self, filename, regex):
+        """Find first pattern occurrence in file"""
+        pattern = re.compile(regex)
+        with open(filename, "r") as f:
+            for line in f:
+                match = pattern.search(line)
+                if match is not None:
+                    return match
+
 
 class StartprojectTest(ProjectTest):
 
@@ -482,6 +492,26 @@ class GenspiderCommandTest(CommandTest):
     def test_same_filename_as_existing_spider_force(self):
         self.test_same_filename_as_existing_spider(force=True)
 
+    def test_url(self, url='test.com', domain="test.com"):
+        self.assertEqual(0, self.call('genspider', '--force', 'test_name', url))
+        self.assertEqual(domain,
+                         self.find_in_file(join(self.proj_mod_path,
+                                                'spiders', 'test_name.py'),
+                                           r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1))
+        self.assertEqual('http://%s/' % domain,
+                         self.find_in_file(join(self.proj_mod_path,
+                                                'spiders', 'test_name.py'),
+                                           r'start_urls\s*=\s*\[\'(.+)\'\]').group(1))
+
+    def test_url_schema(self):
+        self.test_url('http://test.com', 'test.com')
+
+    def test_url_path(self):
+        self.test_url('test.com/some/other/page', 'test.com')
+
+    def test_url_schema_path(self):
+        self.test_url('https://test.com/some/other/page', 'test.com')
+
 
 class GenspiderStandaloneCommandTest(ProjectTest):
 

From 43ea21e8306bc66e8f07abc84cce680726abc7dc Mon Sep 17 00:00:00 2001
From: D R Siddhartha <siddharthadr11@gmail.com>
Date: Tue, 24 Aug 2021 15:18:01 +0530
Subject: [PATCH 455/716] Feed post-processing plugin support (#5190)

---
 .github/workflows/tests-ubuntu.yml  |   2 +-
 docs/topics/feed-exports.rst        |  67 +++-
 scrapy/extensions/feedexport.py     |   4 +
 scrapy/extensions/postprocessing.py | 154 +++++++++
 tests/test_feedexport.py            | 496 ++++++++++++++++++++++++++++
 5 files changed, 721 insertions(+), 2 deletions(-)
 create mode 100644 scrapy/extensions/postprocessing.py

diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 81beda5da..ef1c8362f 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -20,7 +20,7 @@ jobs:
         - python-version: pypy3
           env:
             TOXENV: pypy3
-            PYPY_VERSION: 3.6-v7.3.1
+            PYPY_VERSION: 3.6-v7.3.3
 
         # pinned deps
         - python-version: 3.6.12
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2b3217d62..116967280 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -272,6 +272,7 @@ in multiple files, with the specified maximum item count per file. That way, as
 soon as a file reaches the maximum item count, that file is delivered to the
 feed URI, allowing item delivery to start way before the end of the crawl.
 
+
 .. _item-filter:
 
 Item filtering
@@ -312,6 +313,63 @@ ItemFilter
    :members:
 
 
+.. _post-processing:
+
+Post-Processing
+===============
+
+.. versionadded:: VERSION
+
+Scrapy provides an option to activate plugins to post-process feeds before they are exported
+to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
+can create your own :ref:`plugins <custom-plugins>`. 
+
+These plugins can be activated through the ``postprocessing`` option of a feed.
+The option must be passed a list of post-processing plugins in the order you want
+the feed to be processed. These plugins can be declared either as an import string
+or with the imported class of the plugin. Parameters to plugins can be passed
+through the feed options. See :ref:`feed options <feed-options>` for examples.
+
+.. _builtin-plugins:
+
+Built-in Plugins
+----------------
+
+.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin
+
+.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin
+
+.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin
+
+.. _custom-plugins:
+
+Custom Plugins
+--------------
+
+Each plugin is a class that must implement the following methods:
+
+.. method:: __init__(self, file, feed_options)
+
+    Initialize the plugin.
+
+    :param file: file-like object having at least the `write`, `tell` and `close` methods implemented
+
+    :param feed_options: feed-specific :ref:`options <feed-options>`
+    :type feed_options: :class:`dict`
+
+.. method:: write(self, data)
+
+   Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file.
+   It must return number of bytes written.
+
+.. method:: close(self)
+
+    Close the target file object.
+
+To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You 
+can then access those parameters from the ``__init__`` method of your plugin.
+
+
 Settings
 ========
 
@@ -368,10 +426,12 @@ For instance::
             'encoding': 'latin1',
             'indent': 8,
         },
-        pathlib.Path('items.csv'): {
+        pathlib.Path('items.csv.gz'): {
             'format': 'csv',
             'fields': ['price', 'name'],
             'item_filter': 'myproject.filters.MyCustomFilter2',
+            'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
+            'gzip_compresslevel': 5,
         },
     }
 
@@ -435,6 +495,11 @@ as a fallback value if that key is not provided for a specific feed definition:
 
 -   ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
 
+-   ``postprocessing``: list of :ref:`plugins <post-processing>` to use for post-processing.
+
+    The plugins will be used in the order of the list passed.
+
+    .. versionadded:: VERSION
 
 .. setting:: FEED_EXPORT_ENCODING
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 564c736f2..0f5bf01d0 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -20,6 +20,7 @@ from zope.interface import implementer, Interface
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+from scrapy.extensions.postprocessing import PostProcessingManager
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.conf import feed_complete_default_values_from_settings
 from scrapy.utils.ftp import ftp_store_file
@@ -396,6 +397,9 @@ class FeedExporter:
         """
         storage = self._get_storage(uri, feed_options)
         file = storage.open(spider)
+        if "postprocessing" in feed_options:
+            file = PostProcessingManager(feed_options["postprocessing"], file, feed_options)
+
         exporter = self._get_exporter(
             file=file,
             format=feed_options['format'],
diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py
new file mode 100644
index 000000000..413c2e55e
--- /dev/null
+++ b/scrapy/extensions/postprocessing.py
@@ -0,0 +1,154 @@
+"""
+Extension for processing data before they are exported to feeds.
+"""
+from bz2 import BZ2File
+from gzip import GzipFile
+from io import IOBase
+from lzma import LZMAFile
+from typing import Any, BinaryIO, Dict, List
+
+from scrapy.utils.misc import load_object
+
+
+class GzipPlugin:
+    """
+    Compresses received data using `gzip <https://en.wikipedia.org/wiki/Gzip>`_.
+
+    Accepted ``feed_options`` parameters:
+
+    - `gzip_compresslevel`
+    - `gzip_mtime`
+    - `gzip_filename`
+
+    See :py:class:`gzip.GzipFile` for more info about parameters.
+    """
+
+    def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:
+        self.file = file
+        self.feed_options = feed_options
+        compress_level = self.feed_options.get("gzip_compresslevel", 9)
+        mtime = self.feed_options.get("gzip_mtime")
+        filename = self.feed_options.get("gzip_filename")
+        self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level,
+                                 mtime=mtime, filename=filename)
+
+    def write(self, data: bytes) -> int:
+        return self.gzipfile.write(data)
+
+    def close(self) -> None:
+        self.gzipfile.close()
+        self.file.close()
+
+
+class Bz2Plugin:
+    """
+    Compresses received data using `bz2 <https://en.wikipedia.org/wiki/Bzip2>`_.
+
+    Accepted ``feed_options`` parameters:
+
+    - `bz2_compresslevel`
+
+    See :py:class:`bz2.BZ2File` for more info about parameters.
+    """
+
+    def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:
+        self.file = file
+        self.feed_options = feed_options
+        compress_level = self.feed_options.get("bz2_compresslevel", 9)
+        self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level)
+
+    def write(self, data: bytes) -> int:
+        return self.bz2file.write(data)
+
+    def close(self) -> None:
+        self.bz2file.close()
+        self.file.close()
+
+
+class LZMAPlugin:
+    """
+    Compresses received data using `lzma <https://en.wikipedia.org/wiki/Lempel–Ziv–Markov_chain_algorithm>`_.
+
+    Accepted ``feed_options`` parameters:
+
+    - `lzma_format`
+    - `lzma_check`
+    - `lzma_preset`
+    - `lzma_filters`
+
+    .. note::
+        ``lzma_filters`` cannot be used in pypy version 7.3.1 and older.
+
+    See :py:class:`lzma.LZMAFile` for more info about parameters.
+    """
+
+    def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None:
+        self.file = file
+        self.feed_options = feed_options
+
+        format = self.feed_options.get("lzma_format")
+        check = self.feed_options.get("lzma_check", -1)
+        preset = self.feed_options.get("lzma_preset")
+        filters = self.feed_options.get("lzma_filters")
+        self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format,
+                                 check=check, preset=preset, filters=filters)
+
+    def write(self, data: bytes) -> int:
+        return self.lzmafile.write(data)
+
+    def close(self) -> None:
+        self.lzmafile.close()
+        self.file.close()
+
+
+# io.IOBase is subclassed here, so that exporters can use the PostProcessingManager
+# instance as a file like writable object. This could be needed by some exporters
+# such as CsvItemExporter which wraps the feed storage with io.TextIOWrapper.
+class PostProcessingManager(IOBase):
+    """
+    This will manage and use declared plugins to process data in a
+    pipeline-ish way.
+    :param plugins: all the declared plugins for the feed
+    :type plugins: list
+    :param file: final target file where the processed data will be written
+    :type file: file like object
+    """
+
+    def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None:
+        self.plugins = self._load_plugins(plugins)
+        self.file = file
+        self.feed_options = feed_options
+        self.head_plugin = self._get_head_plugin()
+
+    def write(self, data: bytes) -> int:
+        """
+        Uses all the declared plugins to process data first, then writes
+        the processed data to target file.
+        :param data: data passed to be written to target file
+        :type data: bytes
+        :return: returns number of bytes written
+        :rtype: int
+        """
+        return self.head_plugin.write(data)
+
+    def tell(self) -> int:
+        return self.file.tell()
+
+    def close(self) -> None:
+        """
+        Close the target file along with all the plugins.
+        """
+        self.head_plugin.close()
+
+    def writable(self) -> bool:
+        return True
+
+    def _load_plugins(self, plugins: List[Any]) -> List[Any]:
+        plugins = [load_object(plugin) for plugin in plugins]
+        return plugins
+
+    def _get_head_plugin(self) -> Any:
+        prev = self.file
+        for plugin in self.plugins[::-1]:
+            prev = plugin(prev, self.feed_options)
+        return prev
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 53e6a2018..253f3119c 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1,5 +1,8 @@
+import bz2
 import csv
+import gzip
 import json
+import lzma
 import os
 import random
 import shutil
@@ -1473,6 +1476,499 @@ class FeedExportTest(FeedExportTestBase):
             self.assertEqual(row['expected'], data[feed_options['format']])
 
 
+class FeedPostProcessedExportsTest(FeedExportTestBase):
+    __test__ = True
+
+    items = [{'foo': 'bar'}]
+    expected = b'foo\r\nbar\r\n'
+
+    class MyPlugin1:
+        def __init__(self, file, feed_options):
+            self.file = file
+            self.feed_options = feed_options
+            self.char = self.feed_options.get('plugin1_char', b'')
+
+        def write(self, data):
+            written_count = self.file.write(data)
+            written_count += self.file.write(self.char)
+            return written_count
+
+        def close(self):
+            self.file.close()
+
+    def _named_tempfile(self, name):
+        return os.path.join(self.temp_dir, name)
+
+    @defer.inlineCallbacks
+    def run_and_export(self, spider_cls, settings):
+        """ Run spider with specified settings; return exported data with filename. """
+
+        FEEDS = settings.get('FEEDS') or {}
+        settings['FEEDS'] = {
+            printf_escape(path_to_url(file_path)): feed_options
+            for file_path, feed_options in FEEDS.items()
+        }
+
+        content = {}
+        try:
+            with MockServer() as s:
+                runner = CrawlerRunner(Settings(settings))
+                spider_cls.start_urls = [s.url('/')]
+                yield runner.crawl(spider_cls)
+
+            for file_path, feed_options in FEEDS.items():
+                if not os.path.exists(str(file_path)):
+                    continue
+
+                with open(str(file_path), 'rb') as f:
+                    content[str(file_path)] = f.read()
+
+        finally:
+            for file_path in FEEDS.keys():
+                if not os.path.exists(str(file_path)):
+                    continue
+
+                os.remove(str(file_path))
+
+        return content
+
+    def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=''):
+        data_stream = BytesIO()
+        gzipf = gzip.GzipFile(fileobj=data_stream, filename=filename, mtime=mtime,
+                              compresslevel=compresslevel, mode="wb")
+        gzipf.write(data)
+        gzipf.close()
+        data_stream.seek(0)
+        return data_stream.read()
+
+    @defer.inlineCallbacks
+    def test_gzip_plugin(self):
+
+        filename = self._named_tempfile('gzip_file')
+
+        settings = {
+            'FEEDS': {
+                filename: {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+        try:
+            gzip.decompress(data[filename])
+        except OSError:
+            self.fail("Received invalid gzip data.")
+
+    @defer.inlineCallbacks
+    def test_gzip_plugin_compresslevel(self):
+
+        filename_to_compressed = {
+            self._named_tempfile('compresslevel_0'): self.get_gzip_compressed(self.expected, compresslevel=0),
+            self._named_tempfile('compresslevel_9'): self.get_gzip_compressed(self.expected, compresslevel=9),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('compresslevel_0'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    'gzip_compresslevel': 0,
+                    'gzip_mtime': 0,
+                    'gzip_filename': "",
+                },
+                self._named_tempfile('compresslevel_9'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    'gzip_compresslevel': 9,
+                    'gzip_mtime': 0,
+                    'gzip_filename': "",
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = gzip.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_gzip_plugin_mtime(self):
+        filename_to_compressed = {
+            self._named_tempfile('mtime_123'): self.get_gzip_compressed(self.expected, mtime=123),
+            self._named_tempfile('mtime_123456789'): self.get_gzip_compressed(self.expected, mtime=123456789),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('mtime_123'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    'gzip_mtime': 123,
+                    'gzip_filename': "",
+                },
+                self._named_tempfile('mtime_123456789'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    'gzip_mtime': 123456789,
+                    'gzip_filename': "",
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = gzip.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_gzip_plugin_filename(self):
+        filename_to_compressed = {
+            self._named_tempfile('filename_FILE1'): self.get_gzip_compressed(self.expected, filename="FILE1"),
+            self._named_tempfile('filename_FILE2'): self.get_gzip_compressed(self.expected, filename="FILE2"),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('filename_FILE1'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    'gzip_mtime': 0,
+                    'gzip_filename': "FILE1",
+                },
+                self._named_tempfile('filename_FILE2'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'],
+                    'gzip_mtime': 0,
+                    'gzip_filename': "FILE2",
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = gzip.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_lzma_plugin(self):
+
+        filename = self._named_tempfile('lzma_file')
+
+        settings = {
+            'FEEDS': {
+                filename: {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+        try:
+            lzma.decompress(data[filename])
+        except lzma.LZMAError:
+            self.fail("Received invalid lzma data.")
+
+    @defer.inlineCallbacks
+    def test_lzma_plugin_format(self):
+
+        filename_to_compressed = {
+            self._named_tempfile('format_FORMAT_XZ'): lzma.compress(self.expected, format=lzma.FORMAT_XZ),
+            self._named_tempfile('format_FORMAT_ALONE'): lzma.compress(self.expected, format=lzma.FORMAT_ALONE),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('format_FORMAT_XZ'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_format': lzma.FORMAT_XZ,
+                },
+                self._named_tempfile('format_FORMAT_ALONE'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_format': lzma.FORMAT_ALONE,
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = lzma.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_lzma_plugin_check(self):
+
+        filename_to_compressed = {
+            self._named_tempfile('check_CHECK_NONE'): lzma.compress(self.expected, check=lzma.CHECK_NONE),
+            self._named_tempfile('check_CHECK_CRC256'): lzma.compress(self.expected, check=lzma.CHECK_SHA256),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('check_CHECK_NONE'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_check': lzma.CHECK_NONE,
+                },
+                self._named_tempfile('check_CHECK_CRC256'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_check': lzma.CHECK_SHA256,
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = lzma.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_lzma_plugin_preset(self):
+
+        filename_to_compressed = {
+            self._named_tempfile('preset_PRESET_0'): lzma.compress(self.expected, preset=0),
+            self._named_tempfile('preset_PRESET_9'): lzma.compress(self.expected, preset=9),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('preset_PRESET_0'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_preset': 0,
+                },
+                self._named_tempfile('preset_PRESET_9'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_preset': 9,
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = lzma.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_lzma_plugin_filters(self):
+        import sys
+        if "PyPy" in sys.version:
+            # https://foss.heptapod.net/pypy/pypy/-/issues/3527
+            raise unittest.SkipTest("lzma filters doesn't work in PyPy")
+
+        filters = [{'id': lzma.FILTER_LZMA2}]
+        compressed = lzma.compress(self.expected, filters=filters)
+        filename = self._named_tempfile('filters')
+
+        settings = {
+            'FEEDS': {
+                filename: {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'lzma_filters': filters,
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+        self.assertEqual(compressed, data[filename])
+        result = lzma.decompress(data[filename])
+        self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_bz2_plugin(self):
+
+        filename = self._named_tempfile('bz2_file')
+
+        settings = {
+            'FEEDS': {
+                filename: {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+        try:
+            bz2.decompress(data[filename])
+        except OSError:
+            self.fail("Received invalid bz2 data.")
+
+    @defer.inlineCallbacks
+    def test_bz2_plugin_compresslevel(self):
+
+        filename_to_compressed = {
+            self._named_tempfile('compresslevel_1'): bz2.compress(self.expected, compresslevel=1),
+            self._named_tempfile('compresslevel_9'): bz2.compress(self.expected, compresslevel=9),
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('compresslevel_1'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],
+                    'bz2_compresslevel': 1,
+                },
+                self._named_tempfile('compresslevel_9'): {
+                    'format': 'csv',
+                    'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'],
+                    'bz2_compresslevel': 9,
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, compressed in filename_to_compressed.items():
+            result = bz2.decompress(data[filename])
+            self.assertEqual(compressed, data[filename])
+            self.assertEqual(self.expected, result)
+
+    @defer.inlineCallbacks
+    def test_custom_plugin(self):
+        filename = self._named_tempfile('csv_file')
+
+        settings = {
+            'FEEDS': {
+                filename: {
+                    'format': 'csv',
+                    'postprocessing': [self.MyPlugin1],
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+        self.assertEqual(self.expected, data[filename])
+
+    @defer.inlineCallbacks
+    def test_custom_plugin_with_parameter(self):
+
+        expected = b'foo\r\n\nbar\r\n\n'
+        filename = self._named_tempfile('newline')
+
+        settings = {
+            'FEEDS': {
+                filename: {
+                    'format': 'csv',
+                    'postprocessing': [self.MyPlugin1],
+                    'plugin1_char': b'\n'
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+        self.assertEqual(expected, data[filename])
+
+    @defer.inlineCallbacks
+    def test_custom_plugin_with_compression(self):
+
+        expected = b'foo\r\n\nbar\r\n\n'
+
+        filename_to_decompressor = {
+            self._named_tempfile('bz2'): bz2.decompress,
+            self._named_tempfile('lzma'): lzma.decompress,
+            self._named_tempfile('gzip'): gzip.decompress,
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('bz2'): {
+                    'format': 'csv',
+                    'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.Bz2Plugin'],
+                    'plugin1_char': b'\n',
+                },
+                self._named_tempfile('lzma'): {
+                    'format': 'csv',
+                    'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.LZMAPlugin'],
+                    'plugin1_char': b'\n',
+                },
+                self._named_tempfile('gzip'): {
+                    'format': 'csv',
+                    'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
+                    'plugin1_char': b'\n',
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, decompressor in filename_to_decompressor.items():
+            result = decompressor(data[filename])
+            self.assertEqual(expected, result)
+
+    @defer.inlineCallbacks
+    def test_exports_compatibility_with_postproc(self):
+        import marshal
+        import pickle
+        filename_to_expected = {
+            self._named_tempfile('csv'): b'foo\r\nbar\r\n',
+            self._named_tempfile('json'): b'[\n{"foo": "bar"}\n]',
+            self._named_tempfile('jsonlines'): b'{"foo": "bar"}\n',
+            self._named_tempfile('xml'): b'<?xml version="1.0" encoding="utf-8"?>\n'
+                                         b'<items>\n<item><foo>bar</foo></item>\n</items>',
+        }
+
+        settings = {
+            'FEEDS': {
+                self._named_tempfile('csv'): {
+                    'format': 'csv',
+                    'postprocessing': [self.MyPlugin1],
+                    # empty plugin to activate postprocessing.PostProcessingManager
+                },
+                self._named_tempfile('json'): {
+                    'format': 'json',
+                    'postprocessing': [self.MyPlugin1],
+                },
+                self._named_tempfile('jsonlines'): {
+                    'format': 'jsonlines',
+                    'postprocessing': [self.MyPlugin1],
+                },
+                self._named_tempfile('xml'): {
+                    'format': 'xml',
+                    'postprocessing': [self.MyPlugin1],
+                },
+                self._named_tempfile('marshal'): {
+                    'format': 'marshal',
+                    'postprocessing': [self.MyPlugin1],
+                },
+                self._named_tempfile('pickle'): {
+                    'format': 'pickle',
+                    'postprocessing': [self.MyPlugin1],
+                },
+            },
+        }
+
+        data = yield self.exported_data(self.items, settings)
+
+        for filename, result in data.items():
+            if 'pickle' in filename:
+                expected, result = self.items[0], pickle.loads(result)
+            elif 'marshal' in filename:
+                expected, result = self.items[0], marshal.loads(result)
+            else:
+                expected = filename_to_expected[filename]
+            self.assertEqual(expected, result)
+
+
 class BatchDeliveriesTest(FeedExportTestBase):
     __test__ = True
     _file_mark = '_%(batch_time)s_#%(batch_id)02d_'

From 8284de5e7613c47e69d40d4f6a2a1dc846b50dd6 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Tue, 24 Aug 2021 15:15:29 +0500
Subject: [PATCH 456/716] Fix/silence the Pylint messages added in 2.10 (#5235)

---
 pylintrc                                |  1 +
 scrapy/utils/conf.py                    |  2 +-
 scrapy/utils/deprecate.py               |  2 +-
 tests/keys/__init__.py                  |  8 ++++----
 tests/test_downloader_handlers_http2.py |  2 +-
 tests/test_engine.py                    |  2 +-
 tests/test_exporters.py                 |  4 ++--
 tests/test_http2_client_protocol.py     |  2 +-
 tests/test_loader_deprecated.py         |  2 +-
 tests/test_request_cb_kwargs.py         |  2 +-
 tests/test_scheduler.py                 | 10 +++++-----
 tests/test_utils_conf.py                |  4 ++--
 tests/test_utils_misc/__init__.py       |  2 +-
 13 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/pylintrc b/pylintrc
index a44712507..699686e16 100644
--- a/pylintrc
+++ b/pylintrc
@@ -105,6 +105,7 @@ disable=abstract-method,
         unnecessary-lambda,
         unnecessary-pass,
         unreachable,
+        unspecified-encoding,
         unsubscriptable-object,
         unused-argument,
         unused-import,
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index b904c4a03..24873f75d 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -121,7 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings):
     out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
-    out.setdefault("item_export_kwargs", dict())
+    out.setdefault("item_export_kwargs", {})
     if settings["FEED_EXPORT_INDENT"] is None:
         out.setdefault("indent", None)
     else:
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index f5b17416f..ae727464c 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -79,7 +79,7 @@ def create_deprecated_class(
         # for implementation details
         def __instancecheck__(cls, inst):
             return any(cls.__subclasscheck__(c)
-                       for c in {type(inst), inst.__class__})
+                       for c in (type(inst), inst.__class__))
 
         def __subclasscheck__(cls, sub):
             if cls is not DeprecatedClass.deprecated_class:
diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py
index da202be4d..bb4a8e5af 100644
--- a/tests/keys/__init__.py
+++ b/tests/keys/__init__.py
@@ -40,9 +40,9 @@ def generate_keys():
 
     subject = issuer = Name(
         [
-            NameAttribute(NameOID.COUNTRY_NAME, u"IE"),
-            NameAttribute(NameOID.ORGANIZATION_NAME, u"Scrapy"),
-            NameAttribute(NameOID.COMMON_NAME, u"localhost"),
+            NameAttribute(NameOID.COUNTRY_NAME, "IE"),
+            NameAttribute(NameOID.ORGANIZATION_NAME, "Scrapy"),
+            NameAttribute(NameOID.COMMON_NAME, "localhost"),
         ]
     )
     cert = (
@@ -54,7 +54,7 @@ def generate_keys():
         .not_valid_before(datetime.utcnow())
         .not_valid_after(datetime.utcnow() + timedelta(days=10))
         .add_extension(
-            SubjectAlternativeName([DNSName(u"localhost")]),
+            SubjectAlternativeName([DNSName("localhost")]),
             critical=False,
         )
         .sign(key, SHA256(), default_backend())
diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index 53bb4fe92..8c8c30597 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -219,7 +219,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
     certfile = 'keys/localhost.crt'
 
     scheme = 'https'
-    host = u'127.0.0.1'
+    host = '127.0.0.1'
 
     expected_http_proxy_request_body = b'/'
 
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 92bf45f25..fa7d0c8d4 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -152,7 +152,7 @@ class CrawlerRun:
         self.itemerror = []
         self.itemresp = []
         self.headers = {}
-        self.bytes = defaultdict(lambda: list())
+        self.bytes = defaultdict(list)
         self.signals_caught = {}
         self.spider_class = spider_class
 
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 04bae31d3..b263b3475 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -362,14 +362,14 @@ class CsvItemExporterTest(BaseItemExporterTest):
     def test_errors_default(self):
         with self.assertRaises(UnicodeEncodeError):
             self.assertExportResult(
-                item=dict(text=u'W\u0275\u200Brd'),
+                item=dict(text='W\u0275\u200Brd'),
                 expected=None,
                 encoding='windows-1251',
             )
 
     def test_errors_xmlcharrefreplace(self):
         self.assertExportResult(
-            item=dict(text=u'W\u0275\u200Brd'),
+            item=dict(text='W\u0275\u200Brd'),
             include_headers_line=False,
             expected='W&#629;&#8203;rd\r\n',
             encoding='windows-1251',
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 677ede92b..49c83132f 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -201,7 +201,7 @@ class Https2ClientProtocolTestCase(TestCase):
         self.site = Site(root, timeout=None)
 
         # Start server for testing
-        self.hostname = u'localhost'
+        self.hostname = 'localhost'
         context_factory = ssl_context_factory(self.key_file, self.certificate_file)
 
         server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname)
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index 41afa2896..0fd52da5f 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -703,7 +703,7 @@ class DeprecatedUtilityFunctionsTestCase(unittest.TestCase):
             return None
 
         with warnings.catch_warnings(record=True) as w:
-            wrap_loader_context(function, context=dict())
+            wrap_loader_context(function, context={})
 
             assert len(w) == 1
             assert issubclass(w[0].category, ScrapyDeprecationWarning)
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index b68184b87..738502de8 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -57,7 +57,7 @@ class KeywordArgumentsSpider(MockServerSpider):
         },
     }
 
-    checks = list()
+    checks = []
 
     def start_requests(self):
         data = {'key': 'value', 'number': 123}
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 512a7460e..2d4bfa165 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -22,7 +22,7 @@ MockSlot = collections.namedtuple('MockSlot', ['active'])
 
 class MockDownloader:
     def __init__(self):
-        self.slots = dict()
+        self.slots = {}
 
     def _get_slot_key(self, request, spider):
         if Downloader.DOWNLOAD_SLOT in request.meta:
@@ -31,7 +31,7 @@ class MockDownloader:
         return urlparse_cached(request).hostname or ''
 
     def increment(self, slot_key):
-        slot = self.slots.setdefault(slot_key, MockSlot(active=list()))
+        slot = self.slots.setdefault(slot_key, MockSlot(active=[]))
         slot.active.append(1)
 
     def decrement(self, slot_key):
@@ -114,7 +114,7 @@ class BaseSchedulerInMemoryTester(SchedulerHandler):
         for url, priority in _PRIORITIES:
             self.scheduler.enqueue_request(Request(url, priority=priority))
 
-        priorities = list()
+        priorities = []
         while self.scheduler.has_pending_requests():
             priorities.append(self.scheduler.next_request().priority)
 
@@ -167,7 +167,7 @@ class BaseSchedulerOnDiskTester(SchedulerHandler):
         self.close_scheduler()
         self.create_scheduler()
 
-        priorities = list()
+        priorities = []
         while self.scheduler.has_pending_requests():
             priorities.append(self.scheduler.next_request().priority)
 
@@ -259,7 +259,7 @@ class DownloaderAwareSchedulerTestMixin:
             self.close_scheduler()
             self.create_scheduler()
 
-        dequeued_slots = list()
+        dequeued_slots = []
         requests = []
         downloader = self.mock_crawler.engine.downloader
         while self.scheduler.has_pending_requests():
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index dc2560add..a92880626 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -176,7 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "store_empty": True,
             "uri_params": (1, 2, 3, 4),
             "batch_item_count": 2,
-            "item_export_kwargs": dict(),
+            "item_export_kwargs": {},
         })
 
     def test_feed_complete_default_values_from_settings_non_empty(self):
@@ -199,7 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
             "store_empty": True,
             "uri_params": None,
             "batch_item_count": 2,
-            "item_export_kwargs": dict(),
+            "item_export_kwargs": {},
         })
 
 
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index 47d73a2dd..b83c1d6f0 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -27,7 +27,7 @@ class UtilsMiscTestCase(unittest.TestCase):
     def test_load_object_exceptions(self):
         self.assertRaises(ImportError, load_object, 'nomodule999.mod.function')
         self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999')
-        self.assertRaises(TypeError, load_object, dict())
+        self.assertRaises(TypeError, load_object, {})
 
     def test_walk_modules(self):
         mods = walk_modules('tests.test_utils_misc.test_walk_modules')

From ac9175964dda07da1e838ebb063fc8dd95925e0d Mon Sep 17 00:00:00 2001
From: maanijou <19888963+maanijou@users.noreply.github.com>
Date: Sun, 12 Sep 2021 17:59:20 +0200
Subject: [PATCH 457/716] Improve documentation for spider middlewares

---
 docs/topics/spider-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index f0158dc41..f3fb0d5d7 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.Request` objects and :ref:`item object
+        iterable of :class:`~scrapy.Request` objects or :ref:`item object
         <topics-items>`.
 
         If it returns ``None``, Scrapy will continue processing this exception,

From e5998fb8469dff1e2e965826d2b43f9bad0c17ad Mon Sep 17 00:00:00 2001
From: kamran890 <uetlahore.kamran@gmail.com>
Date: Wed, 22 Sep 2021 03:00:18 +0500
Subject: [PATCH 458/716] Document spider.state attribute (#5174)

---
 docs/topics/jobs.rst    | 2 ++
 docs/topics/spiders.rst | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index e49f37a2f..f16d306c7 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command::
 
     scrapy crawl somespider -s JOBDIR=crawls/somespider-1
 
+.. _topics-keeping-persistent-state-between-batches:
+
 Keeping persistent state between batches
 ========================================
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 67b9e2e0e..4d3d32941 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -122,6 +122,11 @@ scrapy.Spider
       send log messages through it as described on
       :ref:`topics-logging-from-spiders`.
 
+   .. attribute:: state
+
+      A dict you can use to persist some spider state between batches.
+      See :ref:`topics-keeping-persistent-state-between-batches` to know more about it.
+
    .. method:: from_crawler(crawler, *args, **kwargs)
 
        This is the class method used by Scrapy to create your spiders.

From 1829dd774ca0f056e97f7c4621575ef9126e4b57 Mon Sep 17 00:00:00 2001
From: "Reza (Milad) Maanijou" <19888963+maanijou@users.noreply.github.com>
Date: Sat, 25 Sep 2021 20:22:09 +0330
Subject: [PATCH 459/716] Update spider-middleware.rst

---
 docs/topics/spider-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index f3fb0d5d7..08be2b03c 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.Request` objects or :ref:`item object
+        iterable of :class:`~scrapy.Request` objects or :ref:`item objects
         <topics-items>`.
 
         If it returns ``None``, Scrapy will continue processing this exception,

From dfdb779756aa1df2059980de02d359e4e93bac55 Mon Sep 17 00:00:00 2001
From: "Reza (Milad) Maanijou" <19888963+maanijou@users.noreply.github.com>
Date: Sun, 26 Sep 2021 12:45:44 +0330
Subject: [PATCH 460/716] Apply review comments

---
 docs/topics/spider-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 08be2b03c..f1373b9ee 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.Request` objects or :ref:`item objects
+        iterable of :class:`~scrapy.Request` or :ref:`item objects
         <topics-items>`.
 
         If it returns ``None``, Scrapy will continue processing this exception,

From 3c57825b0f3a7bb250aec753b09f964f36500e2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sun, 26 Sep 2021 13:41:26 +0200
Subject: [PATCH 461/716] Update docs/topics/spider-middleware.rst

---
 docs/topics/spider-middleware.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index f1373b9ee..73bedf655 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -122,8 +122,8 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.Request` or :ref:`item objects
-        <topics-items>`.
+        iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>` 
+        objects.
 
         If it returns ``None``, Scrapy will continue processing this exception,
         executing any other :meth:`process_spider_exception` in the following

From 74f146bbe0c41e2c21f431c00ff34d6c5d10cb63 Mon Sep 17 00:00:00 2001
From: Deepanshu <73387559+iDeepverma@users.noreply.github.com>
Date: Fri, 1 Oct 2021 01:47:05 +0530
Subject: [PATCH 462/716] Document update URLLENGTH_LIMIT

---
 docs/topics/settings.rst          | 8 ++++++--
 docs/topics/spider-middleware.rst | 1 +
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 2ab2020fa..4d3ae20cc 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1645,8 +1645,12 @@ Default: ``2083``
 
 Scope: ``spidermiddlewares.urllength``
 
-The maximum URL length to allow for crawled URLs. For more information about
-the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
+The maximum URL length to allow for crawled URLs. You can set this to  ``0``  
+to disable :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` for working 
+with URLs longer than the default value. The default limit acts as a stopping condition in case of
+URLs of increasing length, usually caused by a loop.
+For more information about the default value 
+for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
 
 .. setting:: USER_AGENT
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 73bedf655..a0a7b1fb6 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -440,4 +440,5 @@ UrlLengthMiddleware
    settings (see the settings documentation for more info):
 
       * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.
+        If ``0``, then :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is disabled.
 

From 890f884de46602352de48fa844edd9959c62e473 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Fri, 1 Oct 2021 04:50:42 -0300
Subject: [PATCH 463/716] Allow 'callback' key in keyword arguments for request
 callbacks (#5251)

---
 scrapy/core/scraper.py          |  2 +-
 tests/test_request_cb_kwargs.py | 13 ++++++++-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index d6d6f64f9..f40bccbb3 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -156,7 +156,7 @@ class Scraper:
             callback = result.request.callback or spider._parse
             warn_on_generator_with_return_value(spider, callback)
             dfd = defer_succeed(result)
-            dfd.addCallback(callback, **result.request.cb_kwargs)
+            dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs)
         else:  # result is a Failure
             result.request = request
             warn_on_generator_with_return_value(spider, request.errback)
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 738502de8..8b96fe1a1 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -60,7 +60,7 @@ class KeywordArgumentsSpider(MockServerSpider):
     checks = []
 
     def start_requests(self):
-        data = {'key': 'value', 'number': 123}
+        data = {'key': 'value', 'number': 123, 'callback': 'some_callback'}
         yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
         yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
         yield Request(self.mockserver.url('/general_without'), self.parse_general)
@@ -88,7 +88,8 @@ class KeywordArgumentsSpider(MockServerSpider):
         if response.url.endswith('/general_with'):
             self.checks.append(kwargs['key'] == 'value')
             self.checks.append(kwargs['number'] == 123)
-            self.crawler.stats.inc_value('boolean_checks', 2)
+            self.checks.append(kwargs['callback'] == 'some_callback')
+            self.crawler.stats.inc_value('boolean_checks', 3)
         elif response.url.endswith('/general_without'):
             self.checks.append(kwargs == {})
             self.crawler.stats.inc_value('boolean_checks')
@@ -110,7 +111,7 @@ class KeywordArgumentsSpider(MockServerSpider):
         TypeError: parse_takes_less() got an unexpected keyword argument 'number'
         """
 
-    def parse_takes_more(self, response, key, number, other):
+    def parse_takes_more(self, response, key, number, callback, other):
         """
         Should raise
         TypeError: parse_takes_more() missing 1 required positional argument: 'other'
@@ -161,11 +162,13 @@ class CallbackKeywordArgumentsTestCase(TestCase):
         self.assertTrue(
             str(exceptions['takes_less'].exc_info[1]).endswith(
                 "parse_takes_less() got an unexpected keyword argument 'number'"
-            )
+            ),
+            msg="Exception message: " + str(exceptions['takes_less'].exc_info[1]),
         )
         self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
         self.assertTrue(
             str(exceptions['takes_more'].exc_info[1]).endswith(
                 "parse_takes_more() missing 1 required positional argument: 'other'"
-            )
+            ),
+            msg="Exception message: " + str(exceptions['takes_more'].exc_info[1]),
         )

From fbb1236fd6ee283414360f5209b3a569e112c8cb Mon Sep 17 00:00:00 2001
From: Deepanshu verma <73387559+iDeepverma@users.noreply.github.com>
Date: Fri, 1 Oct 2021 18:46:11 +0530
Subject: [PATCH 464/716] Update docs/topics/settings.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

added suggestion

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/settings.rst | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4d3ae20cc..ed8f1f105 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1645,12 +1645,19 @@ Default: ``2083``
 
 Scope: ``spidermiddlewares.urllength``
 
-The maximum URL length to allow for crawled URLs. You can set this to  ``0``  
-to disable :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` for working 
-with URLs longer than the default value. The default limit acts as a stopping condition in case of
-URLs of increasing length, usually caused by a loop.
-For more information about the default value 
-for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
+The maximum URL length to allow for crawled URLs.
+
+This setting can act as a stopping condition in case of URLs of ever-increasing 
+length, which may be caused for example by a programming error either in the 
+target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and 
+:setting:`DEPTH_LIMIT`.
+
+Use ``0`` to allow URLs of any length.
+
+The default value is copied from the `Microsoft Internet Explorer maximum URL 
+length`_, even though this setting exists for different reasons.
+
+.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
 
 .. setting:: USER_AGENT
 

From d91d82b5064c512e741da106b5fb3a398027d5a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 1 Oct 2021 16:31:29 +0200
Subject: [PATCH 465/716] Make Scrapy SFW again

---
 docs/topics/request-response.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index d3e08efd4..42ce22158 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -300,7 +300,7 @@ errors if needed::
             "http://www.httpbin.org/status/404",    # Not found error
             "http://www.httpbin.org/status/500",    # server issue
             "http://www.httpbin.org:12345/",        # non-responding host, timeout expected
-            "http://www.httphttpbinbin.org/",       # DNS error expected
+            "https://example.invalid/",             # DNS error expected
         ]
 
         def start_requests(self):

From de2043f9c1661208de7b73305a8e3a2395d29583 Mon Sep 17 00:00:00 2001
From: Deepanshu <73387559+iDeepverma@users.noreply.github.com>
Date: Fri, 1 Oct 2021 20:20:00 +0530
Subject: [PATCH 466/716] updated docs/topics/spider-middleware.rst

---
 docs/topics/spider-middleware.rst | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index a0a7b1fb6..f27bc79c0 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -440,5 +440,3 @@ UrlLengthMiddleware
    settings (see the settings documentation for more info):
 
       * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.
-        If ``0``, then :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is disabled.
-

From 47533985f4dc7e58895e1a34c3ea88502f83572a Mon Sep 17 00:00:00 2001
From: Peter Morrison <peter.morrison@medlior.com>
Date: Fri, 1 Oct 2021 12:30:14 -0600
Subject: [PATCH 467/716] Document file expiration method in media-pipeline

---
 docs/topics/media-pipeline.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 46bd2859b..10d2ac990 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -383,6 +383,9 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key:
 
 and pipeline class MyPipeline will have expiration time set to 180.
 
+The last modified time from the file is used to determine the age of the file in days, 
+which is then compared to the set expiration time to determine if the file is expired.
+
 .. _topics-images-thumbnails:
 
 Thumbnail generation for images

From de70b3c58b5b4b2f95468218c194b1e4b99a33c4 Mon Sep 17 00:00:00 2001
From: Deepanshu verma <73387559+iDeepverma@users.noreply.github.com>
Date: Sat, 2 Oct 2021 12:12:58 +0530
Subject: [PATCH 468/716] Update spider-middleware.rst

added empty line
---
 docs/topics/spider-middleware.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index f27bc79c0..3545e760b 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -440,3 +440,4 @@ UrlLengthMiddleware
    settings (see the settings documentation for more info):
 
       * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.
+      

From f10880022273c005a6cb6d9e6ff9cc9a370cc375 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sat, 2 Oct 2021 13:25:15 +0200
Subject: [PATCH 469/716] Update spider-middleware.rst

---
 docs/topics/spider-middleware.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 3545e760b..f27bc79c0 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -440,4 +440,3 @@ UrlLengthMiddleware
    settings (see the settings documentation for more info):
 
       * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.
-      

From ef263042d75586e94d74290d1a41c432f7d87bec Mon Sep 17 00:00:00 2001
From: Raihan Nismara <31585789+raihan71@users.noreply.github.com>
Date: Sun, 3 Oct 2021 13:26:20 +0700
Subject: [PATCH 470/716] Using Logo Scrapy in Readme.md

Logo scrapy used in readme.md made looks nicer
---
 README.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.rst b/README.rst
index 5750e2c0f..05f10bb6c 100644
--- a/README.rst
+++ b/README.rst
@@ -1,3 +1,6 @@
+.. image:: /artwork/scrapy-logo.jpg
+   :width: 400px
+   
 ======
 Scrapy
 ======

From b9647b85d3bc9dcefc7d829ce8304bb28f7cc798 Mon Sep 17 00:00:00 2001
From: Ryan Whelchel <rydwhelchel@gmail.com>
Date: Sun, 3 Oct 2021 17:32:38 -0400
Subject: [PATCH 471/716] docs: restructed phrasing for clarity

---
 docs/intro/tutorial.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 438f3d6df..ba27b18bc 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -277,9 +277,9 @@ As an alternative, you could've written:
 >>> response.css('title::text')[0].get()
 'Quotes to Scrape'
 
-However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
-instance avoids an ``IndexError`` and returns ``None`` when it doesn't
-find any element matching the selection.
+Directly accessing an index on a :class:`~scrapy.selector.SelectorList`
+instance could potentially run into an ``IndexError``. It is recommended to use
+``.get()`` directly instead as it avoids such index errors.
 
 There's a lesson here: for most scraping code, you want it to be resilient to
 errors due to things not being found on a page, so that even if some parts fail

From 764cf0178bb7bc346f1c15ed7ab0adcbb75c43b0 Mon Sep 17 00:00:00 2001
From: Ryan Whelchel <rydwhelchel@gmail.com>
Date: Tue, 5 Oct 2021 10:22:57 -0400
Subject: [PATCH 472/716] Update docs/intro/tutorial.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/intro/tutorial.rst | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index ba27b18bc..fa321a770 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -277,9 +277,20 @@ As an alternative, you could've written:
 >>> response.css('title::text')[0].get()
 'Quotes to Scrape'
 
-Directly accessing an index on a :class:`~scrapy.selector.SelectorList`
-instance could potentially run into an ``IndexError``. It is recommended to use
-``.get()`` directly instead as it avoids such index errors.
+Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will 
+raise an :exc:`IndexError` exception if there are no results::
+
+    >>> response.css('noelement')[0].get()
+    Traceback (most recent call last):
+    File "<console>", line 1, in <module>
+    ...
+    IndexError: list index out of range
+
+You might want to use ``.get()`` directly on the 
+:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` 
+if there are no results::
+
+>>> response.css("noelement").get()
 
 There's a lesson here: for most scraping code, you want it to be resilient to
 errors due to things not being found on a page, so that even if some parts fail

From b081f18a2f232a1bfd04c829ae6894cac93a89a1 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 16 Aug 2019 14:53:42 +0500
Subject: [PATCH 473/716] Add http_auth_domain to HttpAuthMiddleware.

---
 docs/topics/downloader-middleware.rst       | 18 ++++-
 scrapy/downloadermiddlewares/httpauth.py    | 21 ++++-
 tests/test_downloadermiddleware_httpauth.py | 87 ++++++++++++++++++++-
 3 files changed, 119 insertions(+), 7 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 99d57bda9..28b019c80 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -323,8 +323,21 @@ HttpAuthMiddleware
     This middleware authenticates all requests generated from certain spiders
     using `Basic access authentication`_ (aka. HTTP auth).
 
-    To enable HTTP authentication from certain spiders, set the ``http_user``
-    and ``http_pass`` attributes of those spiders.
+    To enable HTTP authentication for a spider, set the ``http_user`` and
+    ``http_pass`` spider attributes to the authentication data and the
+    ``http_auth_domain`` spider attribute to the domain which requires this
+    authentication (its subdomains will be also handled in the same way).
+    You can set ``http_auth_domain`` to ``None`` to enable the
+    authentication for all requests but usually this is not needed.
+
+    .. warning::
+        In the previous Scrapy versions HttpAuthMiddleware sent the
+        authentication data with all requests, which is a security problem if
+        the spider makes requests to several different domains. Currently if
+        the ``http_auth_domain`` attribute is not set, the middleware will use
+        the domain of the first request, which will work for some spider but
+        not for others. In the future the middleware will produce an error
+        instead.
 
     Example::
 
@@ -334,6 +347,7 @@ HttpAuthMiddleware
 
             http_user = 'someuser'
             http_pass = 'somepass'
+            http_auth_domain = 'intranet.example.com'
             name = 'intranet.example.com'
 
             # .. rest of the spider code omitted ...
diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py
index 089bf0d85..1bee3e279 100644
--- a/scrapy/downloadermiddlewares/httpauth.py
+++ b/scrapy/downloadermiddlewares/httpauth.py
@@ -3,10 +3,14 @@ HTTP basic auth downloader middleware
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+import warnings
 
 from w3lib.http import basic_auth_header
 
 from scrapy import signals
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.url import url_is_from_any_domain
 
 
 class HttpAuthMiddleware:
@@ -24,8 +28,23 @@ class HttpAuthMiddleware:
         pwd = getattr(spider, 'http_pass', '')
         if usr or pwd:
             self.auth = basic_auth_header(usr, pwd)
+            if not hasattr(spider, 'http_auth_domain'):
+                warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security '
+                              'problems if the spider makes requests to several different domains. http_auth_domain '
+                              'will be set to the domain of the first request, please set it to the correct value '
+                              'explicitly.',
+                              category=ScrapyDeprecationWarning)
+                self.domain_unset = True
+            else:
+                self.domain = spider.http_auth_domain
+                self.domain_unset = False
 
     def process_request(self, request, spider):
         auth = getattr(self, 'auth', None)
         if auth and b'Authorization' not in request.headers:
-            request.headers[b'Authorization'] = auth
+            domain = urlparse_cached(request).hostname
+            if self.domain_unset:
+                self.domain = domain
+                self.domain_unset = False
+            if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
+                request.headers[b'Authorization'] = auth
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index 3381632b0..0362e2018 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -1,13 +1,60 @@
 import unittest
 
+from w3lib.http import basic_auth_header
+
 from scrapy.http import Request
 from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
 from scrapy.spiders import Spider
 
 
+class TestSpiderLegacy(Spider):
+    http_user = 'foo'
+    http_pass = 'bar'
+
+
 class TestSpider(Spider):
     http_user = 'foo'
     http_pass = 'bar'
+    http_auth_domain = 'example.com'
+
+
+class TestSpiderAny(Spider):
+    http_user = 'foo'
+    http_pass = 'bar'
+    http_auth_domain = None
+
+
+class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
+
+    def setUp(self):
+        self.spider = TestSpiderLegacy('foo')
+
+    def test_auth(self):
+        mw = HttpAuthMiddleware()
+        mw.spider_opened(self.spider)
+
+        # initial request, sets the domain and sends the header
+        req = Request('http://example.com/')
+        assert mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+
+        # subsequent request to the same domain, should send the header
+        req = Request('http://example.com/')
+        assert mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+
+        # subsequent request to a different domain, shouldn't send the header
+        req = Request('http://example-noauth.com/')
+        assert mw.process_request(req, self.spider) is None
+        self.assertNotIn('Authorization', req.headers)
+
+    def test_auth_already_set(self):
+        mw = HttpAuthMiddleware()
+        mw.spider_opened(self.spider)
+        req = Request('http://example.com/',
+                      headers=dict(Authorization='Digest 123'))
+        assert mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], b'Digest 123')
 
 
 class HttpAuthMiddlewareTest(unittest.TestCase):
@@ -20,13 +67,45 @@ class HttpAuthMiddlewareTest(unittest.TestCase):
     def tearDown(self):
         del self.mw
 
-    def test_auth(self):
-        req = Request('http://scrapytest.org/')
+    def test_no_auth(self):
+        req = Request('http://example-noauth.com/')
         assert self.mw.process_request(req, self.spider) is None
-        self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==')
+        self.assertNotIn('Authorization', req.headers)
+
+    def test_auth_domain(self):
+        req = Request('http://example.com/')
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+
+    def test_auth_subdomain(self):
+        req = Request('http://foo.example.com/')
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
 
     def test_auth_already_set(self):
-        req = Request('http://scrapytest.org/',
+        req = Request('http://example.com/',
+                      headers=dict(Authorization='Digest 123'))
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], b'Digest 123')
+
+
+class HttpAuthAnyMiddlewareTest(unittest.TestCase):
+
+    def setUp(self):
+        self.mw = HttpAuthMiddleware()
+        self.spider = TestSpiderAny('foo')
+        self.mw.spider_opened(self.spider)
+
+    def tearDown(self):
+        del self.mw
+
+    def test_auth(self):
+        req = Request('http://example.com/')
+        assert self.mw.process_request(req, self.spider) is None
+        self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))
+
+    def test_auth_already_set(self):
+        req = Request('http://example.com/',
                       headers=dict(Authorization='Digest 123'))
         assert self.mw.process_request(req, self.spider) is None
         self.assertEqual(req.headers['Authorization'], b'Digest 123')

From 7ec5f299c42d07768c02970f0a11f018ed790188 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 22 Aug 2019 20:32:56 +0500
Subject: [PATCH 474/716] Small documentation fixes.

---
 docs/topics/downloader-middleware.rst | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 28b019c80..caf44a903 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -328,16 +328,16 @@ HttpAuthMiddleware
     ``http_auth_domain`` spider attribute to the domain which requires this
     authentication (its subdomains will be also handled in the same way).
     You can set ``http_auth_domain`` to ``None`` to enable the
-    authentication for all requests but usually this is not needed.
+    authentication for all requests but you risk leaking your authentication
+    credentials to unrelated domains.
 
     .. warning::
-        In the previous Scrapy versions HttpAuthMiddleware sent the
-        authentication data with all requests, which is a security problem if
-        the spider makes requests to several different domains. Currently if
-        the ``http_auth_domain`` attribute is not set, the middleware will use
-        the domain of the first request, which will work for some spider but
-        not for others. In the future the middleware will produce an error
-        instead.
+        In previous Scrapy versions HttpAuthMiddleware sent the authentication
+        data with all requests, which is a security problem if the spider
+        makes requests to several different domains. Currently if the
+        ``http_auth_domain`` attribute is not set, the middleware will use the
+        domain of the first request, which will work for some spiders but not
+        for others. In the future the middleware will produce an error instead.
 
     Example::
 

From f0105a882df200f71088603fecaeb9d40679c387 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 5 Oct 2021 13:29:06 +0200
Subject: [PATCH 475/716] Cover 2.5.1 in the release notes

---
 docs/news.rst | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 0ea412e75..4b5cbb2da 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,44 @@
 Release notes
 =============
 
+.. _release-2.5.1:
+
+Scrapy 2.5.1 (2021-10-05)
+-------------------------
+
+*   **Security bug fix:**
+
+    If you use
+    :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
+    (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
+    authentication, any request exposes your credentials to the request target.
+
+    To prevent unintended exposure of authentication credentials to unintended
+    domains, you must now additionally set a new, additional spider attribute,
+    ``http_auth_domain``, and point it to the specific domain to which the
+    authentication credentials must be sent.
+
+    If the ``http_auth_domain`` spider attribute is not set, the domain of the
+    first request will be considered the HTTP authentication target, and
+    authentication credentials will only be sent in requests targeting that
+    domain.
+
+    If you need to send the same HTTP authentication credentials to multiple
+    domains, you can use :func:`w3lib.http.basic_auth_header` instead to
+    set the value of the ``Authorization`` header of your requests.
+
+    If you *really* want your spider to send the same HTTP authentication
+    credentials to any domain, set the ``http_auth_domain`` spider attribute
+    to ``None``.
+
+    Finally, if you are a user of `scrapy-splash`_, know that this version of
+    Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
+    need to upgrade scrapy-splash to a greater version for it to continue to
+    work.
+
+.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
+
+
 .. _release-2.5.0:
 
 Scrapy 2.5.0 (2021-04-06)

From 735750c254e6e82af46b4ebbb35e28b8c0a52250 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 5 Oct 2021 21:10:49 +0200
Subject: [PATCH 476/716] Cover 1.8.1 in the release notes

---
 docs/news.rst | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 4b5cbb2da..5e590f027 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1492,6 +1492,44 @@ affect subclasses:
 (:issue:`3884`)
 
 
+.. _release-1.8.1:
+
+Scrapy 1.8.1 (2021-10-05)
+-------------------------
+
+*   **Security bug fix:**
+
+    If you use
+    :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`
+    (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP
+    authentication, any request exposes your credentials to the request target.
+
+    To prevent unintended exposure of authentication credentials to unintended
+    domains, you must now additionally set a new, additional spider attribute,
+    ``http_auth_domain``, and point it to the specific domain to which the
+    authentication credentials must be sent.
+
+    If the ``http_auth_domain`` spider attribute is not set, the domain of the
+    first request will be considered the HTTP authentication target, and
+    authentication credentials will only be sent in requests targeting that
+    domain.
+
+    If you need to send the same HTTP authentication credentials to multiple
+    domains, you can use :func:`w3lib.http.basic_auth_header` instead to
+    set the value of the ``Authorization`` header of your requests.
+
+    If you *really* want your spider to send the same HTTP authentication
+    credentials to any domain, set the ``http_auth_domain`` spider attribute
+    to ``None``.
+
+    Finally, if you are a user of `scrapy-splash`_, know that this version of
+    Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will
+    need to upgrade scrapy-splash to a greater version for it to continue to
+    work.
+
+.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
+
+
 .. _release-1.8.0:
 
 Scrapy 1.8.0 (2019-10-28)

From 6c858cec91b013853a73e6215b74c90b609bc2da Mon Sep 17 00:00:00 2001
From: Laerte <5853172+Laerte@users.noreply.github.com>
Date: Wed, 6 Oct 2021 12:32:04 -0300
Subject: [PATCH 477/716] Cookies: Cast primitive types to str (#5253)

* cast primitive types to str

* add tests
---
 scrapy/downloadermiddlewares/cookies.py    |  4 ++--
 tests/test_downloadermiddleware_cookies.py | 21 +++++++++++++++++++++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index d95ed3d38..0eee8d758 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -80,8 +80,8 @@ class CookiesMiddleware:
                     logger.warning(msg.format(request, cookie, key))
                     return
                 continue
-            if isinstance(cookie[key], str):
-                decoded[key] = cookie[key]
+            if isinstance(cookie[key], (bool, float, int, str)):
+                decoded[key] = str(cookie[key])
             else:
                 try:
                     decoded[key] = cookie[key].decode("utf8")
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index aff8542e9..36021bfbf 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -347,3 +347,24 @@ class CookiesMiddlewareTest(TestCase):
         self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1')
         self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2')
         self.assertCookieValEqual(req3.headers['Cookie'], 'key=')
+
+    def test_primitive_type_cookies(self):
+        # Boolean
+        req1 = Request('http://example.org', cookies={'a': True})
+        assert self.mw.process_request(req1, self.spider) is None
+        self.assertCookieValEqual(req1.headers['Cookie'], b'a=True')
+
+        # Float
+        req2 = Request('http://example.org', cookies={'a': 9.5})
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertCookieValEqual(req2.headers['Cookie'], b'a=9.5')
+
+        # Integer
+        req3 = Request('http://example.org', cookies={'a': 10})
+        assert self.mw.process_request(req3, self.spider) is None
+        self.assertCookieValEqual(req3.headers['Cookie'], b'a=10')
+
+        # String
+        req4 = Request('http://example.org', cookies={'a': 'b'})
+        assert self.mw.process_request(req4, self.spider) is None
+        self.assertCookieValEqual(req4.headers['Cookie'], b'a=b')

From b1cb007b3b8fef6f037cd1abd38fe9da7190ed26 Mon Sep 17 00:00:00 2001
From: MarvinPetzoldt <78762153+MarvinPetzoldt@users.noreply.github.com>
Date: Wed, 6 Oct 2021 19:08:19 +0200
Subject: [PATCH 478/716] Fixed documentation example

---
 docs/topics/exporters.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 8c30122b6..923336769 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -122,7 +122,7 @@ Example::
       class ProductXmlExporter(XmlItemExporter):
 
           def serialize_field(self, field, name, value):
-              if field == 'price':
+              if name == 'price':
                   return f'$ {str(value)}'
               return super().serialize_field(field, name, value)
 

From 029cab72e8a9b20ebb6b9540e9e01747f0e83dba Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Wed, 6 Oct 2021 14:34:09 -0300
Subject: [PATCH 479/716] [CI] fix pypy test (#5264)

---
 tests/test_request_cb_kwargs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 8b96fe1a1..473a93e69 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -105,7 +105,7 @@ class KeywordArgumentsSpider(MockServerSpider):
         self.checks.append(default == 99)
         self.crawler.stats.inc_value('boolean_checks', 4)
 
-    def parse_takes_less(self, response, key):
+    def parse_takes_less(self, response, key, callback):
         """
         Should raise
         TypeError: parse_takes_less() got an unexpected keyword argument 'number'

From d3f1bf79e883fe3662df827ee47cfc93a372ff02 Mon Sep 17 00:00:00 2001
From: Georgiy Zatserklianyi <GeorgeA92@users.noreply.github.com>
Date: Thu, 7 Oct 2021 17:27:20 +0300
Subject: [PATCH 480/716] Use f-strings where appropriate (#5246)

---
 scrapy/__init__.py                        |  2 +-
 scrapy/core/downloader/contextfactory.py  | 12 +++++----
 scrapy/core/downloader/tls.py             |  8 +++---
 scrapy/downloadermiddlewares/httpcache.py |  2 +-
 scrapy/extensions/feedexport.py           | 33 ++++++++++-------------
 scrapy/extensions/httpcache.py            |  4 +--
 scrapy/utils/misc.py                      |  2 +-
 tests/spiders.py                          | 16 +++++------
 tests/test_closespider.py                 |  2 +-
 tests/test_commands.py                    |  4 +--
 tests/test_downloader_handlers_http2.py   |  2 +-
 tests/test_http_request.py                | 19 +++++++------
 12 files changed, 51 insertions(+), 55 deletions(-)

diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 8a8065bf2..396f98219 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -29,7 +29,7 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 # Check minimum required Python version
 if sys.version_info < (3, 6):
-    print("Scrapy %s requires Python 3.6+" % __version__)
+    print(f"Scrapy {__version__} requires Python 3.6+")
     sys.exit(1)
 
 
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 073ef16bf..b5318c7bb 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -135,11 +135,13 @@ def load_context_factory_from_settings(settings, crawler):
             settings=settings,
             crawler=crawler,
         )
-        msg = """
-            '%s' does not accept `method` argument (type OpenSSL.SSL method,\
-            e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
-            Please upgrade your context factory class to handle them or ignore them.""" % (
-            settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
+        msg = (
+            f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept "
+            "a `method` argument (type OpenSSL.SSL method, e.g. "
+            "OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` "
+            "argument and/or a `tls_ciphers` argument. Please, upgrade your "
+            "context factory class to handle them or ignore them."
+        )
         warnings.warn(msg)
 
     return context_factory
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 2b8990b75..19a56d9b6 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
                 verifyHostname(connection, self._hostnameASCII)
             except (CertificateError, VerificationError) as e:
                 logger.warning(
-                    'Remote certificate is not valid for hostname "{}"; {}'.format(
-                        self._hostnameASCII, e))
+                    'Remote certificate is not valid for hostname "%s"; %s',
+                    self._hostnameASCII, e)
 
             except ValueError as e:
                 logger.warning(
                     'Ignoring error while verifying certificate '
-                    'from host "{}" (exception: {})'.format(
-                        self._hostnameASCII, repr(e)))
+                    'from host "%s" (exception: %r)',
+                    self._hostnameASCII, e)
 
 
 DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 62f1c3a29..80ed7ac75 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -70,7 +70,7 @@ class HttpCacheMiddleware:
             self.stats.inc_value('httpcache/miss', spider=spider)
             if self.ignore_missing:
                 self.stats.inc_value('httpcache/ignore', spider=spider)
-                raise IgnoreRequest("Ignored request not in cache: %s" % request)
+                raise IgnoreRequest(f"Ignored request not in cache: {request}")
             return None  # first time request
 
         # Return cached response only if not expired
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 0f5bf01d0..370723368 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -38,11 +38,10 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
         kwargs['feed_options'] = feed_options
     else:
         warnings.warn(
-            "{} does not support the 'feed_options' keyword argument. Add a "
+            f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a "
             "'feed_options' parameter to its signature to remove this "
             "warning. This parameter will become mandatory in a future "
-            "version of Scrapy."
-            .format(builder.__qualname__),
+            "version of Scrapy.",
             category=ScrapyDeprecationWarning
         )
     return builder(*preargs, uri, *args, **kwargs)
@@ -356,32 +355,28 @@ class FeedExporter:
             # properly closed.
             return defer.maybeDeferred(slot.storage.store, slot.file)
         slot.finish_exporting()
-        logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
-        log_args = {'format': slot.format,
-                    'itemcount': slot.itemcount,
-                    'uri': slot.uri}
+        logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
         d = defer.maybeDeferred(slot.storage.store, slot.file)
 
-        # Use `largs=log_args` to copy log_args into function's scope
-        # instead of using `log_args` from the outer scope
         d.addCallback(
-            self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__
+            self._handle_store_success, logmsg, spider, type(slot.storage).__name__
         )
         d.addErrback(
-            self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__
+            self._handle_store_error, logmsg, spider, type(slot.storage).__name__
         )
         return d
 
-    def _handle_store_error(self, f, largs, logfmt, spider, slot_type):
+    def _handle_store_error(self, f, logmsg, spider, slot_type):
         logger.error(
-            logfmt % "Error storing", largs,
+            "Error storing %s", logmsg,
             exc_info=failure_to_exc_info(f), extra={'spider': spider}
         )
         self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
 
-    def _handle_store_success(self, f, largs, logfmt, spider, slot_type):
+    def _handle_store_success(self, f, logmsg, spider, slot_type):
         logger.info(
-            logfmt % "Stored", largs, extra={'spider': spider}
+            "Stored %s", logmsg,
+            extra={'spider': spider}
         )
         self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
 
@@ -474,10 +469,10 @@ class FeedExporter:
         for uri_template, values in self.feeds.items():
             if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
                 logger.error(
-                    '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
+                    '%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT '
                     'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
-                    'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
-                    ''.format(uri_template)
+                    'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count',
+                    uri_template
                 )
                 return False
         return True
@@ -526,7 +521,7 @@ class FeedExporter:
             instance = build_instance(feedcls)
             method_name = '__new__'
         if instance is None:
-            raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))
+            raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None")
         return instance
 
     def _get_uri_params(self, spider, uri_params, slot=None):
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index e0c04b2de..d0ae29b90 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -226,7 +226,7 @@ class DbmCacheStorage:
         dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
         self.db = self.dbmodule.open(dbpath, 'c')
 
-        logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
+        logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
 
     def close_spider(self, spider):
         self.db.close()
@@ -280,7 +280,7 @@ class FilesystemCacheStorage:
         self._open = gzip.open if self.use_gzip else open
 
     def open_spider(self, spider):
-        logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir},
+        logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
                      extra={'spider': spider})
 
     def close_spider(self, spider):
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 51cef1e91..11c4206c2 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -50,7 +50,7 @@ def load_object(path):
             return path
         else:
             raise TypeError("Unexpected argument type, expected string "
-                            "or object, got: %s" % type(path))
+                            f"or object, got: {type(path)}")
 
     try:
         dot = path.rindex('.')
diff --git a/tests/spiders.py b/tests/spiders.py
index 5b45f897e..67dbbbe0f 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -86,7 +86,7 @@ class SimpleSpider(MetaSpider):
         self.start_urls = [url]
 
     def parse(self, response):
-        self.logger.info("Got response %d" % response.status)
+        self.logger.info(f"Got response {response.status}")
 
 
 class AsyncDefSpider(SimpleSpider):
@@ -95,7 +95,7 @@ class AsyncDefSpider(SimpleSpider):
 
     async def parse(self, response):
         await defer.succeed(42)
-        self.logger.info("Got response %d" % response.status)
+        self.logger.info(f"Got response {response.status}")
 
 
 class AsyncDefAsyncioSpider(SimpleSpider):
@@ -105,7 +105,7 @@ class AsyncDefAsyncioSpider(SimpleSpider):
     async def parse(self, response):
         await asyncio.sleep(0.2)
         status = await get_from_asyncio_queue(response.status)
-        self.logger.info("Got response %d" % status)
+        self.logger.info(f"Got response {status}")
 
 
 class AsyncDefAsyncioReturnSpider(SimpleSpider):
@@ -115,7 +115,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
     async def parse(self, response):
         await asyncio.sleep(0.2)
         status = await get_from_asyncio_queue(response.status)
-        self.logger.info("Got response %d" % status)
+        self.logger.info(f"Got response {status}")
         return [{'id': 1}, {'id': 2}]
 
 
@@ -126,7 +126,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
     async def parse(self, response):
         await asyncio.sleep(0.1)
         status = await get_from_asyncio_queue(response.status)
-        self.logger.info("Got response %d" % status)
+        self.logger.info(f"Got response {status}")
         return {"foo": 42}
 
 
@@ -138,7 +138,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
         await asyncio.sleep(0.2)
         req_id = response.meta.get('req_id', 0)
         status = await get_from_asyncio_queue(response.status)
-        self.logger.info("Got response %d, req_id %d" % (status, req_id))
+        self.logger.info(f"Got response {status}, req_id {req_id}")
         if req_id > 0:
             return
         reqs = []
@@ -155,7 +155,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider):
     async def parse(self, response):
         await asyncio.sleep(0.2)
         yield {'foo': 42}
-        self.logger.info("Got response %d" % response.status)
+        self.logger.info(f"Got response {response.status}")
 
 
 class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
@@ -166,7 +166,7 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
         for i in range(10):
             await asyncio.sleep(0.1)
             yield {'foo': i}
-        self.logger.info("Got response %d" % response.status)
+        self.logger.info(f"Got response {response.status}")
 
 
 class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 5ec5e2989..be8adadb3 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -41,7 +41,7 @@ class TestCloseSpider(TestCase):
         yield crawler.crawl(total=1000000, mockserver=self.mockserver)
         reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_errorcount')
-        key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)
+        key = f'spider_exceptions/{crawler.spider.exception_cls.__name__}'
         errorcount = crawler.stats.get_value(key)
         self.assertTrue(errorcount >= close_on)
 
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 086286b3a..75098a77a 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -498,7 +498,7 @@ class GenspiderCommandTest(CommandTest):
                          self.find_in_file(join(self.proj_mod_path,
                                                 'spiders', 'test_name.py'),
                                            r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1))
-        self.assertEqual('http://%s/' % domain,
+        self.assertEqual(f'http://{domain}/',
                          self.find_in_file(join(self.proj_mod_path,
                                                 'spiders', 'test_name.py'),
                                            r'start_urls\s*=\s*\[\'(.+)\'\]').group(1))
@@ -708,7 +708,7 @@ class MySpider(scrapy.Spider):
         ])
         import asyncio
         loop = asyncio.new_event_loop()
-        self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log)
+        self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log)
 
     def test_output(self):
         spider_code = """
diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index 8c8c30597..3a9db3ee5 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -248,7 +248,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
             self.assertEqual(response.url, request.url)
             self.assertEqual(response.body, b'/')
 
-        http_proxy = '%s?noconnect' % self.getURL('')
+        http_proxy = f"{self.getURL('')}?noconnect"
         request = Request('https://example.com', meta={'proxy': http_proxy})
         with self.assertWarnsRegex(
             Warning,
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index b610087bd..579ef9fa2 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1217,18 +1217,17 @@ class FormRequestTest(RequestTest):
                           response, formcss="input[name='abc']")
 
     def test_from_response_valid_form_methods(self):
-        body = """<form action="post.php" method="%s">
-            <input type="hidden" name="one" value="1">
-            </form>"""
+        form_methods = [[method, method] for method in self.request_class.valid_form_methods]
+        form_methods.append(['UNKNOWN', 'GET'])
 
-        for method in self.request_class.valid_form_methods:
-            response = _buildresponse(body % method)
+        for method, expected in form_methods:
+            response = _buildresponse(
+                f'<form action="post.php" method="{method}">'
+                '<input type="hidden" name="one" value="1">'
+                '</form>'
+            )
             r = self.request_class.from_response(response)
-            self.assertEqual(r.method, method)
-
-        response = _buildresponse(body % 'UNKNOWN')
-        r = self.request_class.from_response(response)
-        self.assertEqual(r.method, 'GET')
+            self.assertEqual(r.method, expected)
 
 
 def _buildresponse(body, **kwargs):

From 65d60b9692dc3475b42d14c744d8a5ac0f2b38cf Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Sun, 10 Oct 2021 05:06:36 -0300
Subject: [PATCH 481/716] [docs] add missing parameter to headers_received
 signal (#5270)

---
 docs/topics/signals.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index a67cc1879..63ad3a9ad 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -413,7 +413,7 @@ headers_received
 .. versionadded:: 2.5
 
 .. signal:: headers_received
-.. function:: headers_received(headers, request, spider)
+.. function:: headers_received(headers, body_length, request, spider)
 
     Sent by the HTTP 1.1 and S3 download handlers when the response headers are
     available for a given request, before downloading any additional content.

From 6fbd6f941f00037ae4718805352e0fa86a781e41 Mon Sep 17 00:00:00 2001
From: ankur19 <ankursaikia08@gmail.com>
Date: Sat, 9 Oct 2021 19:09:51 -0400
Subject: [PATCH 482/716] Fix issue#5145

Fix condition for failing tests

set Selector to None on AttributeError

Add test and remove unused imports

Fix imports
---
 scrapy/loader/__init__.py | 5 ++++-
 tests/test_loader.py      | 8 +++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 014951a8e..91337b949 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -83,6 +83,9 @@ class ItemLoader(itemloaders.ItemLoader):
 
     def __init__(self, item=None, selector=None, response=None, parent=None, **context):
         if selector is None and response is not None:
-            selector = self.default_selector_class(response)
+            try:
+                selector = self.default_selector_class(response)
+            except AttributeError:
+                selector = None
         context.update(response=response)
         super().__init__(item=item, selector=selector, parent=parent, **context)
diff --git a/tests/test_loader.py b/tests/test_loader.py
index b0bc82f4e..f7ab1f236 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -4,7 +4,7 @@ import attr
 from itemadapter import ItemAdapter
 from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst
 
-from scrapy.http import HtmlResponse
+from scrapy.http import HtmlResponse, Response
 from scrapy.item import Item, Field
 from scrapy.loader import ItemLoader
 from scrapy.selector import Selector
@@ -304,6 +304,12 @@ class SelectortemLoaderTest(unittest.TestCase):
 
         l.add_css('name', 'div::text')
         self.assertEqual(l.get_output_value('name'), ['Marta'])
+    
+    def test_init_method_with_base_response(self):
+        """Selector should be None after initialization"""
+        response = Response("https://scrapy.org")
+        l = TestItemLoader(response=response)
+        self.assertIs(l.selector, None)
 
     def test_init_method_with_response(self):
         l = TestItemLoader(response=self.response)

From 3a263280bad53a26490381f293a454be6c25ea30 Mon Sep 17 00:00:00 2001
From: "Kian-Meng, Ang" <kianmeng@cpan.org>
Date: Mon, 11 Oct 2021 22:32:42 +0800
Subject: [PATCH 483/716] Fix typos

---
 docs/news.rst                           | 10 +++++-----
 docs/topics/settings.rst                |  4 ++--
 docs/topics/shell.rst                   |  2 +-
 docs/topics/spiders.rst                 |  4 ++--
 docs/versioning.rst                     |  2 +-
 extras/qpsclient.py                     |  2 +-
 scrapy/downloadermiddlewares/retry.py   |  2 +-
 scrapy/exporters.py                     |  2 +-
 scrapy/linkextractors/lxmlhtml.py       |  2 +-
 scrapy/pipelines/files.py               |  8 ++++----
 scrapy/spiders/feed.py                  |  4 ++--
 scrapy/utils/console.py                 |  2 +-
 scrapy/utils/datatypes.py               |  2 +-
 scrapy/utils/defer.py                   |  2 +-
 scrapy/utils/request.py                 |  4 ++--
 sep/sep-001.rst                         |  2 +-
 sep/sep-005.rst                         |  2 +-
 sep/sep-014.rst                         |  2 +-
 sep/sep-021.rst                         |  2 +-
 tests/test_http_response.py             |  4 ++--
 tests/test_request_attribute_binding.py |  8 ++++----
 tests/test_utils_defer.py               |  4 ++--
 tests/test_utils_template.py            |  2 +-
 23 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 5e590f027..509366c17 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1830,7 +1830,7 @@ New features
 *   A new scheduler priority queue,
     ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
     :ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
-    scheduling improvement on crawls targetting multiple web domains, at the
+    scheduling improvement on crawls targeting multiple web domains, at the
     cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
 
 *   A new :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>` attribute
@@ -2868,7 +2868,7 @@ Bug fixes
 - Fix for selected callbacks when using ``CrawlSpider`` with :command:`scrapy parse <parse>`
   (:issue:`2225`).
 - Fix for invalid JSON and XML files when spider yields no items (:issue:`872`).
-- Implement ``flush()`` fpr ``StreamLogger`` avoiding a warning in logs (:issue:`2125`).
+- Implement ``flush()`` for ``StreamLogger`` avoiding a warning in logs (:issue:`2125`).
 
 Refactoring
 ~~~~~~~~~~~
@@ -3731,7 +3731,7 @@ Scrapy 0.24.3 (2014-08-09)
 - adding some xpath tips to selectors docs (:commit:`2d103e0`)
 - fix tests to account for https://github.com/scrapy/w3lib/pull/23 (:commit:`f8d366a`)
 - get_func_args maximum recursion fix #728 (:commit:`81344ea`)
-- Updated input/ouput processor example according to #560. (:commit:`f7c4ea8`)
+- Updated input/output processor example according to #560. (:commit:`f7c4ea8`)
 - Fixed Python syntax in tutorial. (:commit:`db59ed9`)
 - Add test case for tunneling proxy (:commit:`f090260`)
 - Bugfix for leaking Proxy-Authorization header to remote host when using tunneling (:commit:`d8793af`)
@@ -4393,7 +4393,7 @@ Scrapyd changes
 ~~~~~~~~~~~~~~~
 
 - Scrapyd now uses one process per spider
-- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default)
+- It stores one log file per spider run, and rotate them keeping the latest 5 logs per spider (by default)
 - A minimal web ui was added, available at http://localhost:6800 by default
 - There is now a ``scrapy server`` command to start a Scrapyd server of the current project
 
@@ -4429,7 +4429,7 @@ New features and improvements
 - Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195)
 - Support for overriding default request headers per spider (#181)
 - Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186)
-- Splitted Debian package into two packages - the library and the service (#187)
+- Split Debian package into two packages - the library and the service (#187)
 - Scrapy log refactoring (#188)
 - New extension for keeping persistent spider contexts among different runs (#203)
 - Added ``dont_redirect`` request.meta key for avoiding redirects (#233)
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 2ab2020fa..19a549a02 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1566,7 +1566,7 @@ If a reactor is already installed,
 
 :meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
 :exc:`Exception` if the installed reactor does not match the
-:setting:`TWISTED_REACTOR` setting; therfore, having top-level
+:setting:`TWISTED_REACTOR` setting; therefore, having top-level
 :mod:`~twisted.internet.reactor` imports in project files and imported
 third-party libraries will make Scrapy raise :exc:`Exception` when
 it checks which reactor is installed.
@@ -1658,7 +1658,7 @@ Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
 The default User-Agent to use when crawling, unless overridden. This user agent is
 also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
 if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
-there is no overridding User-Agent header specified for the request.
+there is no overriding User-Agent header specified for the request.
 
 
 Settings documented elsewhere:
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 8c90a506c..007e9fc2f 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -99,7 +99,7 @@ Available Shortcuts
     shortcuts
 
 -   ``fetch(url[, redirect=True])`` - fetch a new response from the given URL
-    and update all related objects accordingly. You can optionaly ask for HTTP
+    and update all related objects accordingly. You can optionally ask for HTTP
     3xx redirections to not be followed by passing ``redirect=False``
 
 -   ``fetch(request)`` - fetch a new response from the given request and update
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 4d3d32941..99e74233a 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -372,7 +372,7 @@ CrawlSpider
        described below. If multiple rules match the same link, the first one
        will be used, according to the order they're defined in this attribute.
 
-   This spider also exposes an overrideable method:
+   This spider also exposes an overridable method:
 
    .. method:: parse_start_url(response, **kwargs)
 
@@ -534,7 +534,7 @@ XMLFeedSpider
                 itertag = 'n:url'
                 # ...
 
-    Apart from these new attributes, this spider has the following overrideable
+    Apart from these new attributes, this spider has the following overridable
     methods too:
 
     .. method:: adapt_response(response)
diff --git a/docs/versioning.rst b/docs/versioning.rst
index 57643ea9a..9d02757b0 100644
--- a/docs/versioning.rst
+++ b/docs/versioning.rst
@@ -13,7 +13,7 @@ There are 3 numbers in a Scrapy version: *A.B.C*
   large changes.
 * *B* is the release number. This will include many changes including features
   and things that possibly break backward compatibility, although we strive to
-  keep theses cases at a minimum.
+  keep these cases at a minimum.
 * *C* is the bugfix release number.
 
 Backward-incompatibilities are explicitly mentioned in the :ref:`release notes <news>`,
diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index f9fb70342..28703650d 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -1,5 +1,5 @@
 """
-A spider that generate light requests to meassure QPS troughput
+A spider that generate light requests to meassure QPS throughput
 
 usage:
 
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index f1fdc3858..c6cc7c56d 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -2,7 +2,7 @@
 An extension to retry failed requests that are potentially caused by temporary
 problems such as a connection timeout or HTTP 500 error.
 
-You can change the behaviour of this middleware by modifing the scraping settings:
+You can change the behaviour of this middleware by modifying the scraping settings:
 RETRY_TIMES - how many times to retry a failed page
 RETRY_HTTP_CODES - which HTTP response codes to retry
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index fb4b565cf..36cca2d05 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -30,7 +30,7 @@ class BaseItemExporter:
         self._configure(kwargs, dont_fail=dont_fail)
 
     def _configure(self, options, dont_fail=False):
-        """Configure the exporter by poping options from the ``options`` dict.
+        """Configure the exporter by popping options from the ``options`` dict.
         If dont_fail is set, it won't raise an exception on unexpected options
         (useful for using with keyword arguments in subclasses ``__init__`` methods)
         """
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index e941c4321..b5d2585a8 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -88,7 +88,7 @@ class LxmlParserLinkExtractor:
     def _process_links(self, links):
         """ Normalize and filter extracted links
 
-        The subclass should override it if neccessary
+        The subclass should override it if necessary
         """
         return self._deduplicate_if_needed(links)
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 8766ef66f..5c52c6c28 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -85,7 +85,7 @@ class S3FilesStore:
     AWS_USE_SSL = None
     AWS_VERIFY = None
 
-    POLICY = 'private'  # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
+    POLICY = 'private'  # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
     HEADERS = {
         'Cache-Control': 'max-age=172800',
     }
@@ -142,7 +142,7 @@ class S3FilesStore:
             **extra)
 
     def _headers_to_botocore_kwargs(self, headers):
-        """ Convert headers to botocore keyword agruments.
+        """ Convert headers to botocore keyword arguments.
         """
         # This is required while we need to support both boto and botocore.
         mapping = CaselessDict({
@@ -190,7 +190,7 @@ class GCSFilesStore:
     CACHE_CONTROL = 'max-age=172800'
 
     # The bucket's default object ACL will be applied to the object.
-    # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
+    # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
     POLICY = None
 
     def __init__(self, uri):
@@ -291,7 +291,7 @@ class FilesPipeline(MediaPipeline):
     """Abstract pipeline that implement the file downloading
 
     This pipeline tries to minimize network transfers and file processing,
-    doing stat of the files and determining if file is new, uptodate or
+    doing stat of the files and determining if file is new, up-to-date or
     expired.
 
     ``new`` files are those that pipeline never processed and needs to be
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 6ed17e4dd..bef2d6b24 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -43,7 +43,7 @@ class XMLFeedSpider(Spider):
         return response
 
     def parse_node(self, response, selector):
-        """This method must be overriden with your custom spider functionality"""
+        """This method must be overridden with your custom spider functionality"""
         if hasattr(self, 'parse_item'):  # backward compatibility
             return self.parse_item(response, selector)
         raise NotImplementedError
@@ -113,7 +113,7 @@ class CSVFeedSpider(Spider):
         return response
 
     def parse_row(self, response, row):
-        """This method must be overriden with your custom spider functionality"""
+        """This method must be overridden with your custom spider functionality"""
         raise NotImplementedError
 
     def parse_rows(self, response):
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 133261fd7..1bc0bd45f 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -14,7 +14,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
     @wraps(_embed_ipython_shell)
     def wrapper(namespace=namespace, banner=''):
         config = load_default_config()
-        # Always use .instace() to ensure _instance propagation to all parents
+        # Always use .instance() to ensure _instance propagation to all parents
         # this is needed for <TAB> completion works well for new imports
         # and clear the instance to always have the fresh env
         # on repeated breaks like with inspect_response()
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index e31284a7f..47df8a717 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -41,7 +41,7 @@ class CaselessDict(dict):
         return key.lower()
 
     def normvalue(self, value):
-        """Method to normalize values prior to be setted"""
+        """Method to normalize values prior to be set"""
         return value
 
     def get(self, key, def_val=None):
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index b317c12a3..b02bfdccb 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -34,7 +34,7 @@ def defer_succeed(result) -> Deferred:
     """Same as twisted.internet.defer.succeed but delay calling callback until
     next reactor loop
 
-    It delays by 100ms so reactor has a chance to go trough readers and writers
+    It delays by 100ms so reactor has a chance to go through readers and writers
     before attending pending delayed calls, so do not set delay to zero.
     """
     from twisted.internet import reactor
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 57dcc5f2c..70ef3ba2b 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -48,7 +48,7 @@ def request_fingerprint(
     the fingerprint.
 
     For this reason, request headers are ignored by default when calculating
-    the fingeprint. If you want to include specific headers use the
+    the fingerprint. If you want to include specific headers use the
     include_headers argument, which is a list of Request headers to include.
 
     Also, servers usually ignore fragments in urls when handling requests,
@@ -78,7 +78,7 @@ def request_fingerprint(
 
 
 def request_authenticate(request: Request, username: str, password: str) -> None:
-    """Autenticate the given request (in place) using the HTTP basic access
+    """Authenticate the given request (in place) using the HTTP basic access
     authentication mechanism (RFC 2617) and the given username and password
     """
     request.headers['Authorization'] = basic_auth_header(username, password)
diff --git a/sep/sep-001.rst b/sep/sep-001.rst
index 00226283f..f704e113f 100644
--- a/sep/sep-001.rst
+++ b/sep/sep-001.rst
@@ -260,7 +260,7 @@ ItemForm
    ia['width'] = x.x('//p[@class="width"]')
    ia['volume'] = x.x('//p[@class="volume"]')
 
-   # another example passing parametes on instance
+   # another example passing parameters on instance
    ia = NewsForm(response, encoding='utf-8')
    ia['name'] = x.x('//p[@class="name"]')
 
diff --git a/sep/sep-005.rst b/sep/sep-005.rst
index e795838e4..08ed367b3 100644
--- a/sep/sep-005.rst
+++ b/sep/sep-005.rst
@@ -107,7 +107,7 @@ gUsing default_builder
 
 
 This will use default_builder as the builder for every field in the item class.
-As a reducer is not set reducers will be set based on Item Field classess.
+As a reducer is not set reducers will be set based on Item Field classes.
 
 gReset default_builder for a field
 ==================================
diff --git a/sep/sep-014.rst b/sep/sep-014.rst
index 8ca81824d..0859e3f7c 100644
--- a/sep/sep-014.rst
+++ b/sep/sep-014.rst
@@ -64,7 +64,7 @@ Request Processors takes requests objects and can perform any action to them,
 like filtering or modifying on the fly.
 
 The current ``LinkExtractor`` had integrated link processing, like
-canonicalize. Request Processors can be reutilized and applied in serie.
+canonicalize. Request Processors can be reutilized and applied in series.
 
 Request Generator
 -----------------
diff --git a/sep/sep-021.rst b/sep/sep-021.rst
index 372429791..c1ec16f7f 100644
--- a/sep/sep-021.rst
+++ b/sep/sep-021.rst
@@ -22,7 +22,7 @@ Instead, the hooks are spread over:
 * Downloader handlers (DOWNLOADER_HANDLERS)
 * Item pipelines (ITEM_PIPELINES)
 * Feed exporters and storages (FEED_EXPORTERS, FEED_STORAGES)
-* Overrideable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc)
+* Overridable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc)
 * Generic extensions (EXTENSIONS)
 * CLI commands (COMMANDS_MODULE)
 
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index c376a46cd..0ec5257e1 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -19,7 +19,7 @@ class BaseResponseTest(unittest.TestCase):
     response_class = Response
 
     def test_init(self):
-        # Response requires url in the consturctor
+        # Response requires url in the constructor
         self.assertRaises(Exception, self.response_class)
         self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class))
         self.assertRaises(TypeError, self.response_class, b"http://example.com")
@@ -392,7 +392,7 @@ class TextResponseTest(BaseResponseTest):
     def test_declared_encoding_invalid(self):
         """Check that unknown declared encodings are ignored"""
         r = self.response_class("http://www.example.com",
-                                headers={"Content-type": ["text/html; charset=UKNOWN"]},
+                                headers={"Content-type": ["text/html; charset=UNKNOWN"]},
                                 body=b"\xc2\xa3")
         self.assertEqual(r._declared_encoding(), None)
         self._assert_response_values(r, 'utf-8', "\xa3")
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index 00c532c41..25d9657d5 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -106,9 +106,9 @@ class CrawlTestCase(TestCase):
         """
         Downloader middleware which returns a response with an specific 'request' attribute.
 
-        * The spider callback should receive the overriden response.request
-        * Handlers listening to the response_received signal should receive the overriden response.request
-        * The "crawled" log message should show the overriden response.request
+        * The spider callback should receive the overridden response.request
+        * Handlers listening to the response_received signal should receive the overridden response.request
+        * The "crawled" log message should show the overridden response.request
         """
         signal_params = {}
 
@@ -144,7 +144,7 @@ class CrawlTestCase(TestCase):
         An exception is raised but caught by the next middleware, which
         returns a Response with a specific 'request' attribute.
 
-        The spider callback should receive the overriden response.request
+        The spider callback should receive the overridden response.request
         """
         url = self.mockserver.url("/status?n=200")
         runner = CrawlerRunner(settings={
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 7a5f458c7..032dbc8c5 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -23,7 +23,7 @@ class MustbeDeferredTest(unittest.TestCase):
 
         dfd = mustbe_deferred(_append, 1)
         dfd.addCallback(self.assertEqual, [1, 2])  # it is [1] with maybeDeferred
-        steps.append(2)  # add another value, that should be catched by assertEqual
+        steps.append(2)  # add another value, that should be caught by assertEqual
         return dfd
 
     def test_unfired_deferred(self):
@@ -37,7 +37,7 @@ class MustbeDeferredTest(unittest.TestCase):
 
         dfd = mustbe_deferred(_append, 1)
         dfd.addCallback(self.assertEqual, [1, 2])  # it is [1] with maybeDeferred
-        steps.append(2)  # add another value, that should be catched by assertEqual
+        steps.append(2)  # add another value, that should be caught by assertEqual
         return dfd
 
 
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 5ff2e41ef..1d5e63363 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -36,7 +36,7 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
             self.assertEqual(result.read().decode('utf8'), rendered)
 
         os.remove(render_path)
-        assert not os.path.exists(render_path)  # Failure of test iself
+        assert not os.path.exists(render_path)  # Failure of test itself
 
 
 if '__main__' == __name__:

From d08199f631814bdaadafc441bb47cbe71ced2d8d Mon Sep 17 00:00:00 2001
From: Jake Herbst <jmherbst@gmail.com>
Date: Tue, 12 Oct 2021 13:20:09 -0400
Subject: [PATCH 484/716] Removing unnecessary line from docs to prevent test
 failure (#5274)

---
 docs/intro/tutorial.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index fa321a770..ca5856881 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -282,7 +282,6 @@ raise an :exc:`IndexError` exception if there are no results::
 
     >>> response.css('noelement')[0].get()
     Traceback (most recent call last):
-    File "<console>", line 1, in <module>
     ...
     IndexError: list index out of range
 

From e5b057cfd4472e970b9b51757a1b823b2f585b09 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 13 Oct 2021 19:06:51 +0500
Subject: [PATCH 485/716] Don't use a HTTP request in a case that will not be
 awaited.

---
 tests/test_utils_asyncgen.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index 41993a934..d9e6bc2eb 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -1,3 +1,4 @@
+from twisted.internet.defer import Deferred
 from twisted.trial import unittest
 
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal
@@ -31,7 +32,10 @@ async def process_iterable(iterable):
 async def process_iterable_awaiting(iterable):
     async for i in iterable:
         yield i * 2
-        await get_web_client_agent_req('http://example.com')
+        d = Deferred()
+        from twisted.internet import reactor
+        reactor.callLater(0, d.callback, 42)
+        await d
 
 
 class ProcessIterableUniversalTest(unittest.TestCase):

From a642b73e1a1ba355bae9d5b9d1e87a9da0696293 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 13 Oct 2021 19:21:49 +0500
Subject: [PATCH 486/716] Remove an unused import.

---
 tests/test_utils_asyncgen.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index d9e6bc2eb..7abe17c22 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -3,7 +3,6 @@ from twisted.trial import unittest
 
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal
 from scrapy.utils.defer import deferred_f_from_coro_f
-from scrapy.utils.test import get_web_client_agent_req
 
 
 class AsyncgenUtilsTest(unittest.TestCase):

From 3243aa2cd54c8789eb5667998aa8296f6adaa9a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=B0=AD=E4=B9=9D=E9=BC=8E?= <109224573@qq.com>
Date: Thu, 14 Oct 2021 10:18:26 +0800
Subject: [PATCH 487/716] docs: fix typo

---
 docs/topics/loaders.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index c0f534493..0d63700c8 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -56,7 +56,7 @@ chapter <topics-items>`::
         l.add_xpath('name', '//div[@class="product_name"]')
         l.add_xpath('name', '//div[@class="product_title"]')
         l.add_xpath('price', '//p[@id="price"]')
-        l.add_css('stock', 'p#stock]')
+        l.add_css('stock', 'p#stock')
         l.add_value('last_updated', 'today') # you can also use literal values
         return l.load_item()
 

From ca320feb2afa5eae3990ed21df6a8df930e8cf9f Mon Sep 17 00:00:00 2001
From: Erik Kemperman <erikkemperman@gmail.com>
Date: Fri, 15 Oct 2021 15:43:55 +0200
Subject: [PATCH 488/716] Add LOG_FILE_APPEND to settings

---
 docs/topics/logging.rst             |  5 ++++-
 docs/topics/settings.rst            | 10 ++++++++++
 scrapy/settings/default_settings.py |  1 +
 scrapy/utils/log.py                 |  3 ++-
 tests/test_crawler.py               | 29 ++++++++++++++++++++++++++++-
 5 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index dda04dc4d..d593c74c6 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -143,6 +143,7 @@ Logging settings
 These settings can be used to configure the logging:
 
 * :setting:`LOG_FILE`
+* :setting:`LOG_FILE_APPEND`
 * :setting:`LOG_ENABLED`
 * :setting:`LOG_ENCODING`
 * :setting:`LOG_LEVEL`
@@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If
 :setting:`LOG_FILE` is set, messages sent through the root logger will be
 redirected to a file named :setting:`LOG_FILE` with encoding
 :setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log
-messages will be displayed on the standard error. Lastly, if
+messages will be displayed on the standard error. If :setting:`LOG_FILE` is set
+and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten
+(discarding the output from previous runs, if any). Lastly, if
 :setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.
 
 :setting:`LOG_LEVEL` determines the minimum level of severity to display, those
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index e63aca312..210c1def7 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1026,6 +1026,16 @@ Default: ``None``
 
 File name to use for logging output. If ``None``, standard error will be used.
 
+.. setting:: LOG_FILE_APPEND
+
+LOG_FILE_APPEND
+---------------
+
+Default: ``True``
+
+If ``False``, the log file specified with :setting:`LOG_FILE` will be
+overwritten (discarding the output from previous runs, if any).
+
 .. setting:: LOG_FORMAT
 
 LOG_FORMAT
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 4ef330dd2..8389a70cb 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -207,6 +207,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
 LOG_STDOUT = False
 LOG_LEVEL = 'DEBUG'
 LOG_FILE = None
+LOG_FILE_APPEND = True
 LOG_SHORT_NAMES = False
 
 SCHEDULER_DEBUG = False
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 6c456ed60..0441c0358 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -124,8 +124,9 @@ def _get_handler(settings):
     """ Return a log handler object according to settings """
     filename = settings.get('LOG_FILE')
     if filename:
+        mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w'
         encoding = settings.get('LOG_ENCODING')
-        handler = logging.FileHandler(filename, encoding=encoding)
+        handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
     elif settings.getbool('LOG_ENABLED'):
         handler = logging.StreamHandler()
     else:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index dec517bb6..a80ad4388 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -98,6 +98,8 @@ class CrawlerLoggingTestCase(unittest.TestCase):
 
     def test_spider_custom_settings_log_level(self):
         log_file = self.mktemp()
+        with open(log_file, 'wb') as fo:
+            fo.write('previous message\n'.encode('utf-8'))
 
         class MySpider(scrapy.Spider):
             name = 'spider'
@@ -119,8 +121,9 @@ class CrawlerLoggingTestCase(unittest.TestCase):
         logging.error('error message')
 
         with open(log_file, 'rb') as fo:
-            logged = fo.read().decode('utf8')
+            logged = fo.read().decode('utf-8')
 
+        self.assertIn('previous message', logged)
         self.assertNotIn('debug message', logged)
         self.assertIn('info message', logged)
         self.assertIn('warning message', logged)
@@ -131,6 +134,30 @@ class CrawlerLoggingTestCase(unittest.TestCase):
             crawler.stats.get_value('log_count/INFO') - info_count, 1)
         self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)
 
+    def test_spider_custom_settings_log_append(self):
+        log_file = self.mktemp()
+        with open(log_file, 'wb') as fo:
+            fo.write('previous message\n'.encode('utf-8'))
+
+        class MySpider(scrapy.Spider):
+            name = 'spider'
+            custom_settings = {
+                'LOG_FILE': log_file,
+                'LOG_FILE_APPEND': False,
+                # disable telnet if not available to avoid an extra warning
+                'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
+            }
+
+        configure_logging()
+        crawler = Crawler(MySpider, {})
+        logging.debug('debug message')
+
+        with open(log_file, 'rb') as fo:
+            logged = fo.read().decode('utf-8')
+
+        self.assertNotIn('previous message', logged)
+        self.assertIn('debug message', logged)
+
 
 class SpiderLoaderWithWrongInterface:
 

From 98ee3ddffeeabdc896c3f2cffce8310ebfc97570 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 15 Oct 2021 17:18:46 +0200
Subject: [PATCH 489/716] Freeze flake8

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index e274fc8d2..07552ba8d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -57,6 +57,7 @@ deps =
     # Twisted[http2] is required to import some files
     Twisted[http2]>=17.9.0
     pytest-flake8
+    flake8==3.9.2  # https://github.com/tholo/pytest-flake8/issues/81
 commands =
     py.test --flake8 {posargs:docs scrapy tests}
 

From d774d6a9c46cb9697d8fe64a55da9ae321be7539 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 15 Oct 2021 17:25:22 +0200
Subject: [PATCH 490/716] Remove unused variable

---
 tests/test_crawler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index a80ad4388..be067155e 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -149,7 +149,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
             }
 
         configure_logging()
-        crawler = Crawler(MySpider, {})
+        Crawler(MySpider, {})
         logging.debug('debug message')
 
         with open(log_file, 'rb') as fo:

From aec7146e2f870f5e8f0f58bd596b1ec40c64b3c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Raphael=20Tom=C3=A9=20Santana?= <raphaelts3@gmail.com>
Date: Fri, 15 Oct 2021 20:38:53 -0300
Subject: [PATCH 491/716] Add how Scrapy is pronounced to the docs

---
 docs/intro/overview.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index dd80c7bd0..d75f7f636 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -4,7 +4,7 @@
 Scrapy at a glance
 ==================
 
-Scrapy is an application framework for crawling web sites and extracting
+Scrapy (pronounced SKRAY-peye /ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting
 structured data which can be used for a wide range of useful applications, like
 data mining, information processing or historical archival.
 

From 027ecd8686d7da74b73412e4aa38d8be36b5b9f1 Mon Sep 17 00:00:00 2001
From: raphaelts3 <raphaelts3@gmail.com>
Date: Sat, 16 Oct 2021 10:52:54 -0300
Subject: [PATCH 492/716] Update docs/intro/overview.rst

Co-authored-by: azzamsa <17734314+azzamsa@users.noreply.github.com>
---
 docs/intro/overview.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index d75f7f636..405bf845d 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -4,7 +4,7 @@
 Scrapy at a glance
 ==================
 
-Scrapy (pronounced SKRAY-peye /ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting
+Scrapy (/ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting
 structured data which can be used for a wide range of useful applications, like
 data mining, information processing or historical archival.
 

From cfff79cee6a97528185b7d24e2b660b99c07945f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Mon, 18 Oct 2021 17:09:17 -0300
Subject: [PATCH 493/716] Make Python 3.10 support official (#5265)

---
 .github/workflows/checks.yml       |  8 ++++----
 .github/workflows/publish.yml      |  4 ++--
 .github/workflows/tests-macos.yml  |  2 +-
 .github/workflows/tests-ubuntu.yml | 14 ++++++--------
 .readthedocs.yml                   |  9 +++++----
 setup.py                           |  1 +
 tox.ini                            |  1 +
 7 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 6bdfcb5dc..80df9469d 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -8,10 +8,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.9
+        - python-version: "3.10"
           env:
             TOXENV: security
-        - python-version: 3.9
+        - python-version: "3.10"
           env:
             TOXENV: flake8
         # Pylint requires installing reppy, which does not support Python 3.9
@@ -20,10 +20,10 @@ jobs:
           env:
             TOXENV: pylint
             TOX_PIP_VERSION: 20.3.3
-        - python-version: 3.9
+        - python-version: 3.6
           env:
             TOXENV: typing
-        - python-version: 3.8  # Keep in sync with .readthedocs.yml
+        - python-version: "3.10"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs
 
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index b48066ea4..44b682830 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -9,10 +9,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python 3.9
+    - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.9
+        python-version: "3.10"
 
     - name: Check Tag
       id: check-release-tag
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 095ca1013..3aaf688c7 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
 
     steps:
     - uses: actions/checkout@v2
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index ef1c8362f..5ea50e644 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -17,6 +17,12 @@ jobs:
         - python-version: 3.9
           env:
             TOXENV: py
+        - python-version: "3.10"
+          env:
+            TOXENV: py
+        - python-version: "3.10"
+          env:
+            TOXENV: asyncio
         - python-version: pypy3
           env:
             TOXENV: pypy3
@@ -42,14 +48,6 @@ jobs:
             TOXENV: extra-deps
             TOX_PIP_VERSION: 20.3.3
 
-        # 3.10-pre
-        - python-version: "3.10.0-beta.4"
-          env:
-            TOXENV: py
-        - python-version: "3.10.0-beta.4"
-          env:
-            TOXENV: asyncio
-
     steps:
     - uses: actions/checkout@v2
 
diff --git a/.readthedocs.yml b/.readthedocs.yml
index 80a1cd036..390be3749 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -5,12 +5,13 @@ sphinx:
   fail_on_warning: true
 
 build:
-  image: latest
+  os: ubuntu-20.04
+  tools:
+    # For available versions, see:
+    # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
+    python: "3.10"  # Keep in sync with .github/workflows/checks.yml
 
 python:
-  # For available versions, see:
-  # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
-  version: 3.8  # Keep in sync with .github/workflows/checks.yml
   install:
     - requirements: docs/requirements.txt
     - path: .
diff --git a/setup.py b/setup.py
index ed2b6e347..3a6ff2836 100644
--- a/setup.py
+++ b/setup.py
@@ -87,6 +87,7 @@ setup(
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Internet :: WWW/HTTP',
diff --git a/tox.ini b/tox.ini
index 07552ba8d..021dd9988 100644
--- a/tox.ini
+++ b/tox.ini
@@ -41,6 +41,7 @@ deps =
     types-pyOpenSSL==20.0.3
     types-setuptools==57.0.0
 commands =
+    pip install types-dataclasses  # remove once py36 support is dropped
     mypy --show-error-codes {posargs: scrapy tests}
 
 [testenv:security]

From 144d1eb8341c427fa1fae109db3e9487f255d3fe Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Fri, 22 Oct 2021 21:46:01 +0500
Subject: [PATCH 494/716] Add Deferred-to-Future helpers (#5288)

---
 conftest.py                   |  6 ++++
 docs/topics/asyncio.rst       | 17 ++++++++++
 docs/topics/coroutines.rst    | 13 +++++---
 docs/topics/item-pipeline.rst |  4 ++-
 pytest.ini                    |  1 +
 scrapy/utils/defer.py         | 63 +++++++++++++++++++++++++++++++++--
 tests/test_pipelines.py       | 30 ++++++++++++++++-
 7 files changed, 126 insertions(+), 8 deletions(-)

diff --git a/conftest.py b/conftest.py
index 05b4ccdad..117087790 100644
--- a/conftest.py
+++ b/conftest.py
@@ -75,6 +75,12 @@ def only_asyncio(request, reactor_pytest):
         pytest.skip('This test is only run with --reactor=asyncio')
 
 
+@pytest.fixture(autouse=True)
+def only_not_asyncio(request, reactor_pytest):
+    if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
+        pytest.skip('This test is only run without --reactor=asyncio')
+
+
 def pytest_configure(config):
     if config.getoption("--reactor") == "asyncio":
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 82c5f271f..28241ae24 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -39,5 +39,22 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the
 :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
 use it instead of the default asyncio event loop.
 
+.. _asyncio-await-dfd:
 
+Awaiting on Deferreds
+=====================
 
+When the asyncio reactor isn't installed, you can await on Deferreds in the
+coroutines directly. When it is installed, this is not possible anymore, due to
+specifics of the Scrapy coroutine integration (the coroutines are wrapped into
+:class:`asyncio.Future` objects, not into
+:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into
+Futures. Scrapy provides two helpers for this:
+
+.. autofunction:: scrapy.utils.defer.deferred_to_future
+.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future
+.. tip:: If you need to use these functions in code that aims to be compatible
+         with lower versions of Scrapy that do not provide these functions,
+         down to Scrapy 2.0 (earlier versions do not support
+         :mod:`asyncio`), you can copy the implementation of these functions
+         into your own code.
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 0904637b0..2aef755c7 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -75,23 +75,28 @@ coroutines, functions that return Deferreds and functions that return
 :term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
 This means you can use many useful Python libraries providing such code::
 
-    class MySpider(Spider):
+    class MySpiderDeferred(Spider):
         # ...
-        async def parse_with_deferred(self, response):
+        async def parse(self, response):
             additional_response = await treq.get('https://additional.url')
             additional_data = await treq.content(additional_response)
             # ... use response and additional_data to yield items and requests
 
-        async def parse_with_asyncio(self, response):
+    class MySpiderAsyncio(Spider):
+        # ...
+        async def parse(self, response):
             async with aiohttp.ClientSession() as session:
                 async with session.get('https://additional.url') as additional_response:
-                    additional_data = await r.text()
+                    additional_data = await additional_response.text()
             # ... use response and additional_data to yield items and requests
 
 .. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
           :mod:`asyncio` loop and to use them you need to
           :doc:`enable asyncio support in Scrapy<asyncio>`.
 
+.. note:: If you want to ``await`` on Deferreds while using the asyncio reactor,
+          you need to :ref:`wrap them<asyncio-await-dfd>`.
+
 Common use cases for asynchronous code include:
 
 * requesting data from websites, databases and other services (in callbacks,
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 5351a2293..391751364 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -190,6 +190,8 @@ item.
 
     import scrapy
     from itemadapter import ItemAdapter
+    from scrapy.utils.defer import maybe_deferred_to_future
+
 
     class ScreenshotPipeline:
         """Pipeline that uses Splash to render screenshot of
@@ -202,7 +204,7 @@ item.
             encoded_item_url = quote(adapter["url"])
             screenshot_url = self.SPLASH_URL.format(encoded_item_url)
             request = scrapy.Request(screenshot_url)
-            response = await spider.crawler.engine.download(request, spider)
+            response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider))
 
             if response.status != 200:
                 # Error happened, return item.
diff --git a/pytest.ini b/pytest.ini
index 6de08c78d..fa5d6b34f 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -20,5 +20,6 @@ addopts =
     --ignore=docs/utils
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
+    only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
 filterwarnings=
     ignore::DeprecationWarning:twisted.web.test.test_webclient
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index b02bfdccb..d7adc0a77 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -3,9 +3,16 @@ Helper functions for dealing with Twisted deferreds
 """
 import asyncio
 import inspect
-from collections.abc import Coroutine
+from asyncio import Future
 from functools import wraps
-from typing import Any, Callable, Generator, Iterable
+from typing import (
+    Any,
+    Callable,
+    Coroutine,
+    Generator,
+    Iterable,
+    Union
+)
 
 from twisted.internet import defer
 from twisted.internet.defer import Deferred, DeferredList, ensureDeferred
@@ -171,3 +178,55 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
         return defer.fail(result)
     else:
         return defer.succeed(result)
+
+
+def deferred_to_future(d: Deferred) -> Future:
+    """
+    .. versionadded:: VERSION
+
+    Return an :class:`asyncio.Future` object that wraps *d*.
+
+    When :ref:`using the asyncio reactor <install-asyncio>`, you cannot await
+    on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy
+    callables defined as coroutines <coroutine-support>`, you can only await on
+    ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects
+    allows you to wait on them::
+
+        class MySpider(Spider):
+            ...
+            async def parse(self, response):
+                d = treq.get('https://example.com/additional')
+                additional_response = await deferred_to_future(d)
+    """
+    return d.asFuture(asyncio.get_event_loop())
+
+
+def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
+    """
+    .. versionadded:: VERSION
+
+    Return *d* as an object that can be awaited from a :ref:`Scrapy callable
+    defined as a coroutine <coroutine-support>`.
+
+    What you can await in Scrapy callables defined as coroutines depends on the
+    value of :setting:`TWISTED_REACTOR`:
+
+    -   When not using the asyncio reactor, you can only await on
+        :class:`~twisted.internet.defer.Deferred` objects.
+
+    -   When :ref:`using the asyncio reactor <install-asyncio>`, you can only
+        await on :class:`asyncio.Future` objects.
+
+    If you want to write code that uses ``Deferred`` objects but works with any
+    reactor, use this function on all ``Deferred`` objects::
+
+        class MySpider(Spider):
+            ...
+            async def parse(self, response):
+                d = treq.get('https://example.com/additional')
+                extra_response = await maybe_deferred_to_future(d)
+    """
+    if not is_asyncio_reactor_installed():
+        return d
+    else:
+        return deferred_to_future(d)
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index ff3af9a74..8e432b913 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -6,6 +6,7 @@ from twisted.internet.defer import Deferred
 from twisted.trial import unittest
 
 from scrapy import Spider, signals, Request
+from scrapy.utils.defer import maybe_deferred_to_future, deferred_to_future
 from scrapy.utils.test import get_crawler, get_from_asyncio_queue
 
 from tests.mockserver import MockServer
@@ -31,18 +32,38 @@ class DeferredPipeline:
 
 class AsyncDefPipeline:
     async def process_item(self, item, spider):
-        await defer.succeed(42)
+        d = Deferred()
+        from twisted.internet import reactor
+        reactor.callLater(0, d.callback, None)
+        await maybe_deferred_to_future(d)
         item['pipeline_passed'] = True
         return item
 
 
 class AsyncDefAsyncioPipeline:
     async def process_item(self, item, spider):
+        d = Deferred()
+        from twisted.internet import reactor
+        reactor.callLater(0, d.callback, None)
+        await deferred_to_future(d)
         await asyncio.sleep(0.2)
         item['pipeline_passed'] = await get_from_asyncio_queue(True)
         return item
 
 
+class AsyncDefNotAsyncioPipeline:
+    async def process_item(self, item, spider):
+        d1 = Deferred()
+        from twisted.internet import reactor
+        reactor.callLater(0, d1.callback, None)
+        await d1
+        d2 = Deferred()
+        reactor.callLater(0, d2.callback, None)
+        await maybe_deferred_to_future(d2)
+        item['pipeline_passed'] = True
+        return item
+
+
 class ItemSpider(Spider):
     name = 'itemspider'
 
@@ -99,3 +120,10 @@ class PipelineTestCase(unittest.TestCase):
         crawler = self._create_crawler(AsyncDefAsyncioPipeline)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(len(self.items), 1)
+
+    @mark.only_not_asyncio()
+    @defer.inlineCallbacks
+    def test_asyncdef_not_asyncio_pipeline(self):
+        crawler = self._create_crawler(AsyncDefNotAsyncioPipeline)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertEqual(len(self.items), 1)

From 51adf71b1b4ae703bb1cb561882c710eedac2359 Mon Sep 17 00:00:00 2001
From: azzamsa <me@azzamsa.com>
Date: Sun, 24 Oct 2021 10:52:56 +0700
Subject: [PATCH 495/716] refactor: use `pytest` command as the recommended
 entry point

`pytest` is recommended command since pytest 3.0.
There is a possibility for `py.test` to be deprecated or even removed.

https://github.com/pytest-dev/pytest/issues/1629
---
 tox.ini | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tox.ini b/tox.ini
index 021dd9988..e4514f512 100644
--- a/tox.ini
+++ b/tox.ini
@@ -29,7 +29,7 @@ passenv =
 #allow tox virtualenv to upgrade pip/wheel/setuptools
 download = true
 commands =
-    py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
+    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
 install_command =
     pip install -U -ctests/upper-constraints.txt {opts} {packages}
 
@@ -60,7 +60,7 @@ deps =
     pytest-flake8
     flake8==3.9.2  # https://github.com/tholo/pytest-flake8/issues/81
 commands =
-    py.test --flake8 {posargs:docs scrapy tests}
+    pytest --flake8 {posargs:docs scrapy tests}
 
 [testenv:pylint]
 basepython = python3
@@ -142,7 +142,7 @@ setenv =
 [testenv:pypy3]
 basepython = pypy3
 commands =
-    py.test {posargs:--durations=10 docs scrapy tests}
+    pytest {posargs:--durations=10 docs scrapy tests}
 
 [testenv:pypy3-pinned]
 basepython = {[testenv:pypy3]basepython}

From 67994d1dddcba4c1fe53dd7bdf7b978d1733ae1b Mon Sep 17 00:00:00 2001
From: azzamsa <me@azzamsa.com>
Date: Wed, 27 Oct 2021 21:55:05 +0700
Subject: [PATCH 496/716] fix: `CodeBlockParser` has been renamed to
 `PythonCodeBlockParser`

---
 docs/conftest.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/conftest.py b/docs/conftest.py
index 8c735e838..a0636f8ac 100644
--- a/docs/conftest.py
+++ b/docs/conftest.py
@@ -3,7 +3,11 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
 
 from scrapy.http.response.html import HtmlResponse
 from sybil import Sybil
-from sybil.parsers.codeblock import CodeBlockParser
+try:
+    # >2.0.1
+    from sybil.parsers.codeblock import PythonCodeBlockParser
+except ImportError:
+    from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser
 from sybil.parsers.doctest import DocTestParser
 from sybil.parsers.skip import skip
 
@@ -21,7 +25,7 @@ def setup(namespace):
 pytest_collect_file = Sybil(
     parsers=[
         DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
-        CodeBlockParser(future_imports=['print_function']),
+        PythonCodeBlockParser(future_imports=['print_function']),
         skip,
     ],
     pattern='*.rst',

From 55cce25a799ab0ed9d5edd60ff0f35988318e9b9 Mon Sep 17 00:00:00 2001
From: azzamsa <me@azzamsa.com>
Date: Mon, 25 Oct 2021 21:14:11 +0700
Subject: [PATCH 497/716] test: `test_format_engine_status`

---
 tests/test_crawl.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 84bac9b50..7bda3bef2 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -274,6 +274,28 @@ with multiples lines
         self.assertEqual(s['engine.spider.name'], crawler.spider.name)
         self.assertEqual(s['len(engine.scraper.slot.active)'], 1)
 
+    @defer.inlineCallbacks
+    def test_format_engine_status(self):
+        from scrapy.utils.engine import format_engine_status
+        est = []
+
+        def cb(response):
+            est.append(format_engine_status(crawler.engine))
+
+        crawler = self.runner.create_crawler(SingleRequestSpider)
+        yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
+        self.assertEqual(len(est), 1, est)
+        est = est[0].split("\n")[2:-2]  # remove header & footer
+        # convert to dict
+        est = [x.split(":") for x in est]
+        est = [x for sublist in est for x in sublist]  # flatten
+        est = [x.lstrip().rstrip() for x in est]
+        it = iter(est)
+        s = dict(zip(it, it))
+
+        self.assertEqual(s['engine.spider.name'], crawler.spider.name)
+        self.assertEqual(s['len(engine.scraper.slot.active)'], '1')
+
     @defer.inlineCallbacks
     def test_graceful_crawl_error_handling(self):
         """

From 28eba610e22c0d2a42e830b4e64746edf44598f9 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Mon, 15 Nov 2021 12:24:54 +0500
Subject: [PATCH 498/716] Re-enable Windows tests for Python 3.9 and 3.10.
 (#5316)

---
 .github/workflows/tests-windows.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 30fda33e8..6fabf5cde 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -17,10 +17,12 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: py
-        # https://twistedmatrix.com/trac/ticket/9990
-        #- python-version: 3.9
-          #env:
-            #TOXENV: py
+        - python-version: 3.9
+          env:
+            TOXENV: py
+        - python-version: "3.10"
+          env:
+            TOXENV: py
 
     steps:
     - uses: actions/checkout@v2

From f2c800c5c9f88b4b583181e9cf49eb3cd8d538f0 Mon Sep 17 00:00:00 2001
From: Samuel Marchal <samuel.marchal@wakeo.co>
Date: Mon, 15 Nov 2021 11:14:54 +0100
Subject: [PATCH 499/716] Improve open_in_browser base tag injection (#5319)

---
 scrapy/utils/response.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index b3ef7b463..8b109dced 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -3,8 +3,9 @@ This module provides some useful functions for working with
 scrapy.http.Response objects
 """
 import os
-import webbrowser
+import re
 import tempfile
+import webbrowser
 from typing import Any, Callable, Iterable, Optional, Tuple, Union
 from weakref import WeakKeyDictionary
 
@@ -80,8 +81,9 @@ def open_in_browser(
     body = response.body
     if isinstance(response, HtmlResponse):
         if b'<base' not in body:
-            repl = f'<head><base href="{response.url}">'
-            body = body.replace(b'<head>', to_bytes(repl))
+            repl = fr'\1<base href="{response.url}">'
+            body = re.sub(b"<!--.*?-->", b"", body, flags=re.DOTALL)
+            body = re.sub(rb"(<head(?:>|\s.*?>))", to_bytes(repl), body)
         ext = '.html'
     elif isinstance(response, TextResponse):
         ext = '.txt'

From 75ed765476a2ac66ea8f52e7b29186864f65535c Mon Sep 17 00:00:00 2001
From: Samuel Marchal <samuel.marchal@wakeo.co>
Date: Mon, 15 Nov 2021 14:31:24 +0100
Subject: [PATCH 500/716] Test coverage for open_in_browser base tag injection
 (#5319)

---
 tests/test_utils_response.py | 53 ++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index d6f4c0bb5..0a09f6109 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase):
         self.assertEqual(response_status_message(200), '200 OK')
         self.assertEqual(response_status_message(404), '404 Not Found')
         self.assertEqual(response_status_message(573), "573 Unknown Status")
+
+    def test_inject_base_url(self):
+        url = "http://www.example.com"
+
+        def check_base_url(burl):
+            path = urlparse(burl).path
+            if not os.path.exists(path):
+                path = burl.replace('file://', '')
+            with open(path, "rb") as f:
+                bbody = f.read()
+            self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
+            return True
+
+        r1 = HtmlResponse(url, body=b"""
+        <html>
+            <head><title>Dummy</title></head>
+            <body><p>Hello world.</p></body>
+        </html>""")
+        r2 = HtmlResponse(url, body=b"""
+        <html>
+            <head id="foo"><title>Dummy</title></head>
+            <body>Hello world.</body>
+        </html>""")
+        r3 = HtmlResponse(url, body=b"""
+        <html>
+            <head><title>Dummy</title></head>
+            <body>
+                <header>Hello header</header>
+                <p>Hello world.</p>
+            </body>
+        </html>""")
+        r4 = HtmlResponse(url, body=b"""
+        <html>
+            <!-- <head>Dummy comment</head> -->
+            <head><title>Dummy</title></head>
+            <body><p>Hello world.</p></body>
+        </html>""")
+        r5 = HtmlResponse(url, body=b"""
+        <html>
+            <!--[if IE]>
+            <head><title>IE head</title></head>
+            <![endif]-->
+            <!--[if !IE]>-->
+            <head><title>Standard head</title></head>
+            <!--<![endif]-->
+            <body><p>Hello world.</p></body>
+        </html>""")
+
+        assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
+        assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head"
+        assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag"
+        assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment"
+        assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment"

From c316ca45a5b1b19622c96049c9378d8c45adba60 Mon Sep 17 00:00:00 2001
From: Alex <alexpdev@protonmail.com>
Date: Tue, 16 Nov 2021 01:20:56 -0800
Subject: [PATCH 501/716] Use augmented assignment statements (#5322)

---
 scrapy/core/downloader/handlers/http11.py | 2 +-
 scrapy/core/http2/stream.py               | 4 ++--
 tests/test_request_left.py                | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 8a91d4c5e..38935667d 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -213,7 +213,7 @@ class TunnelingAgent(Agent):
         # proxy host and port are required for HTTP pool `key`
         # otherwise, same remote host connection request could reuse
         # a cached tunneled connection to a different proxy
-        key = key + self._proxyConf
+        key += self._proxyConf
         return super()._requestWithEndpoint(
             key=key,
             endpoint=endpoint,
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index c2a4b702f..5c393c027 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -285,8 +285,8 @@ class Stream:
 
             self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False)
 
-            bytes_to_send_size = bytes_to_send_size - chunk_size
-            self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size
+            bytes_to_send_size -= chunk_size
+            self.metadata['remaining_content_length'] -= chunk_size
 
         self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length'])
 
diff --git a/tests/test_request_left.py b/tests/test_request_left.py
index 373b2e49c..4d4483881 100644
--- a/tests/test_request_left.py
+++ b/tests/test_request_left.py
@@ -22,7 +22,7 @@ class SignalCatcherSpider(Spider):
         return spider
 
     def on_request_left(self, request, spider):
-        self.caught_times = self.caught_times + 1
+        self.caught_times += 1
 
 
 class TestCatching(TestCase):

From 6ec66c96fb962a276c2d285cd5ffa34d84925df1 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 26 Nov 2021 12:25:45 +0500
Subject: [PATCH 502/716] Fix and pin pylint.

---
 pylintrc | 1 +
 tox.ini  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/pylintrc b/pylintrc
index 699686e16..2cdd6321e 100644
--- a/pylintrc
+++ b/pylintrc
@@ -112,6 +112,7 @@ disable=abstract-method,
         unused-private-member,
         unused-variable,
         unused-wildcard-import,
+        use-implicit-booleaness-not-comparison,
         used-before-assignment,
         useless-object-inheritance,  # Required for Python 2 support
         useless-return,
diff --git a/tox.ini b/tox.ini
index e4514f512..2031a2d92 100644
--- a/tox.ini
+++ b/tox.ini
@@ -66,7 +66,7 @@ commands =
 basepython = python3
 deps =
     {[testenv:extra-deps]deps}
-    pylint
+    pylint==2.12.1
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 

From 4cc039628eb4861f98f7997e90125745e30f8687 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 26 Nov 2021 19:52:03 +0500
Subject: [PATCH 503/716] Fix typing of middleware methods.

---
 scrapy/core/downloader/middleware.py |  5 ++++-
 scrapy/core/spidermw.py              |  3 ++-
 scrapy/middleware.py                 | 16 ++++++++++------
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index a5619d8a4..289147466 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -3,7 +3,7 @@ Downloader Middleware manager
 
 See documentation in docs/topics/downloader-middleware.rst
 """
-from typing import Callable, Union
+from typing import Callable, Union, cast
 
 from twisted.internet import defer
 from twisted.python.failure import Failure
@@ -37,6 +37,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         @defer.inlineCallbacks
         def process_request(request: Request):
             for method in self.methods['process_request']:
+                method = cast(Callable, method)
                 response = yield deferred_from_coro(method(request=request, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput(
@@ -55,6 +56,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 return response
 
             for method in self.methods['process_response']:
+                method = cast(Callable, method)
                 response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
                 if not isinstance(response, (Response, Request)):
                     raise _InvalidOutput(
@@ -69,6 +71,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         def process_exception(failure: Failure):
             exception = failure.value
             for method in self.methods['process_exception']:
+                method = cast(Callable, method)
                 response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput(
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 7e58521ac..7cdc28284 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -4,7 +4,7 @@ Spider Middleware manager
 See documentation in docs/topics/spider-middleware.rst
 """
 from itertools import islice
-from typing import Any, Callable, Generator, Iterable, Union
+from typing import Any, Callable, Generator, Iterable, Union, cast
 
 from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure
@@ -47,6 +47,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
     def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request,
                               spider: Spider) -> Any:
         for method in self.methods['process_spider_input']:
+            method = cast(Callable, method)
             try:
                 result = method(response=response, spider=spider)
                 if result is not None:
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index bbec38086..e8f60287a 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,7 +1,7 @@
 import logging
 import pprint
 from collections import defaultdict, deque
-from typing import Callable, Deque, Dict
+from typing import Callable, Deque, Dict, Optional, cast, Iterable
 
 from twisted.internet.defer import Deferred
 
@@ -21,7 +21,8 @@ class MiddlewareManager:
 
     def __init__(self, *middlewares):
         self.middlewares = middlewares
-        self.methods: Dict[str, Deque[Callable]] = defaultdict(deque)
+        # Optional because process_spider_output and process_spider_exception can be None
+        self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque)
         for mw in middlewares:
             self._add_middleware(mw)
 
@@ -64,14 +65,17 @@ class MiddlewareManager:
             self.methods['close_spider'].appendleft(mw.close_spider)
 
     def _process_parallel(self, methodname: str, obj, *args) -> Deferred:
-        return process_parallel(self.methods[methodname], obj, *args)
+        methods = cast(Iterable[Callable], self.methods[methodname])
+        return process_parallel(methods, obj, *args)
 
     def _process_chain(self, methodname: str, obj, *args) -> Deferred:
-        return process_chain(self.methods[methodname], obj, *args)
+        methods = cast(Iterable[Callable], self.methods[methodname])
+        return process_chain(methods, obj, *args)
 
     def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred:
-        return process_chain_both(self.methods[cb_methodname],
-                                  self.methods[eb_methodname], obj, *args)
+        cb_methods = cast(Iterable[Callable], self.methods[cb_methodname])
+        eb_methods = cast(Iterable[Callable], self.methods[eb_methodname])
+        return process_chain_both(cb_methods, eb_methods, obj, *args)
 
     def open_spider(self, spider: Spider) -> Deferred:
         return self._process_parallel('open_spider', spider)

From eb62906c3e4c1e1f8e3e6c7965a04d5e65c61907 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 1 Dec 2021 17:40:41 +0500
Subject: [PATCH 504/716] Extract utils.log.log_reactor_info().

---
 scrapy/utils/log.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 0441c0358..9887ecc40 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -143,7 +143,7 @@ def _get_handler(settings):
     return handler
 
 
-def log_scrapy_info(settings):
+def log_scrapy_info(settings: Settings) -> None:
     logger.info("Scrapy %(version)s started (bot: %(bot)s)",
                 {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
     versions = [
@@ -152,6 +152,10 @@ def log_scrapy_info(settings):
         if name != "Scrapy"
     ]
     logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
+    log_reactor_info()
+
+
+def log_reactor_info() -> None:
     from twisted.internet import reactor
     logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
     from twisted.internet import asyncioreactor

From 6483dfdbe17cd66c409435b95a05850a3c94b5ee Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 1 Dec 2021 19:53:39 +0500
Subject: [PATCH 505/716] Move install_shutdown_handlers() from __init__() to
 start().

---
 scrapy/crawler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 578016536..357f14dc0 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -278,7 +278,6 @@ class CrawlerProcess(CrawlerRunner):
 
     def __init__(self, settings=None, install_root_handler=True):
         super().__init__(settings)
-        install_shutdown_handlers(self._signal_shutdown)
         configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)
 
@@ -318,6 +317,7 @@ class CrawlerProcess(CrawlerRunner):
                 return
             d.addBoth(self._stop_reactor)
 
+        install_shutdown_handlers(self._signal_shutdown)
         resolver_class = load_object(self.settings["DNS_RESOLVER"])
         resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
         resolver.install_on_reactor()

From d6a384b3cfdb36cd19b32942479487a9e47c244b Mon Sep 17 00:00:00 2001
From: yogender26 <95638485+yogender26@users.noreply.github.com>
Date: Thu, 23 Dec 2021 04:09:05 +0530
Subject: [PATCH 506/716] corrrection of coma (#5347)

---
 scrapy/commands/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 6e77551c6..5f1dabd33 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -43,14 +43,14 @@ class ScrapyCommand:
 
     def long_desc(self):
         """A long description of the command. Return short description when not
-        available. It cannot contain newlines, since contents will be formatted
+        available. It cannot contain newlines since contents will be formatted
         by optparser which removes newlines and wraps text.
         """
         return self.short_desc()
 
     def help(self):
         """An extensive help for the command. It will be shown when using the
-        "help" command. It can contain newlines, since no post-formatting will
+        "help" command. It can contain newlines since no post-formatting will
         be applied to its contents.
         """
         return self.long_desc()

From 46ef9cf771789f1db513bbf2f65243d3320ce695 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 22 Dec 2021 21:24:59 +0500
Subject: [PATCH 507/716] Don't install non-working shutdown handlers in
 `scrapy shell`.

---
 scrapy/commands/shell.py | 2 +-
 scrapy/crawler.py        | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index d1944df3d..de81986d8 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -75,6 +75,6 @@ class Command(ScrapyCommand):
 
     def _start_crawler_thread(self):
         t = Thread(target=self.crawler_process.start,
-                   kwargs={'stop_after_crawl': False})
+                   kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
         t.daemon = True
         t.start()
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 357f14dc0..e54ad9750 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -297,7 +297,7 @@ class CrawlerProcess(CrawlerRunner):
                     {'signame': signame})
         reactor.callFromThread(self._stop_reactor)
 
-    def start(self, stop_after_crawl=True):
+    def start(self, stop_after_crawl=True, install_signal_handlers=True):
         """
         This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
         size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
@@ -308,6 +308,9 @@ class CrawlerProcess(CrawlerRunner):
 
         :param bool stop_after_crawl: stop or not the reactor when all
             crawlers have finished
+
+        :param bool install_signal_handlers: whether to install the shutdown
+            handlers (default: True)
         """
         from twisted.internet import reactor
         if stop_after_crawl:
@@ -317,7 +320,8 @@ class CrawlerProcess(CrawlerRunner):
                 return
             d.addBoth(self._stop_reactor)
 
-        install_shutdown_handlers(self._signal_shutdown)
+        if install_signal_handlers:
+            install_shutdown_handlers(self._signal_shutdown)
         resolver_class = load_object(self.settings["DNS_RESOLVER"])
         resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
         resolver.install_on_reactor()

From 60c8838554a79e70c22a7c6a57baedfcaf521444 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Dec 2021 16:07:18 +0500
Subject: [PATCH 508/716] Move installing the reactor from CrawlerProcess to
 Crawler.

---
 scrapy/crawler.py     | 31 ++++++++++++++++++++-----------
 scrapy/utils/log.py   |  1 -
 tests/test_crawler.py |  5 ++++-
 3 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index e54ad9750..95cfb1bd1 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -25,6 +25,7 @@ from scrapy.utils.log import (
     configure_logging,
     get_scrapy_root_handler,
     install_scrapy_root_handler,
+    log_reactor_info,
     log_scrapy_info,
     LogCounterHandler,
 )
@@ -38,7 +39,7 @@ logger = logging.getLogger(__name__)
 
 class Crawler:
 
-    def __init__(self, spidercls, settings=None):
+    def __init__(self, spidercls, settings=None, init_reactor: bool = False):
         if isinstance(spidercls, Spider):
             raise ValueError('The spidercls argument must be a class, not an object')
 
@@ -69,6 +70,19 @@ class Crawler:
 
         lf_cls = load_object(self.settings['LOG_FORMATTER'])
         self.logformatter = lf_cls.from_crawler(self)
+
+        if init_reactor:
+            # this needs to be done after the spider settings are merged,
+            # but before something imports twisted.internet.reactor
+            if self.settings.get("TWISTED_REACTOR"):
+                install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
+            else:
+                from twisted.internet import default
+                default.install()
+            log_reactor_info()
+        if self.settings.get("TWISTED_REACTOR"):
+            verify_installed_reactor(self.settings["TWISTED_REACTOR"])
+
         self.extensions = ExtensionManager.from_crawler(self)
 
         self.settings.freeze()
@@ -153,7 +167,6 @@ class CrawlerRunner:
         self._crawlers = set()
         self._active = set()
         self.bootstrap_failed = False
-        self._handle_twisted_reactor()
 
     @property
     def spiders(self):
@@ -247,10 +260,6 @@ class CrawlerRunner:
         while self._active:
             yield defer.DeferredList(self._active)
 
-    def _handle_twisted_reactor(self):
-        if self.settings.get("TWISTED_REACTOR"):
-            verify_installed_reactor(self.settings["TWISTED_REACTOR"])
-
 
 class CrawlerProcess(CrawlerRunner):
     """
@@ -297,6 +306,11 @@ class CrawlerProcess(CrawlerRunner):
                     {'signame': signame})
         reactor.callFromThread(self._stop_reactor)
 
+    def _create_crawler(self, spidercls):
+        if isinstance(spidercls, str):
+            spidercls = self.spider_loader.load(spidercls)
+        return Crawler(spidercls, self.settings, init_reactor=True)
+
     def start(self, stop_after_crawl=True, install_signal_handlers=True):
         """
         This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
@@ -341,8 +355,3 @@ class CrawlerProcess(CrawlerRunner):
             reactor.stop()
         except RuntimeError:  # raised if already stopped or in shutdown stage
             pass
-
-    def _handle_twisted_reactor(self):
-        if self.settings.get("TWISTED_REACTOR"):
-            install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
-        super()._handle_twisted_reactor()
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 9887ecc40..78e302d19 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -152,7 +152,6 @@ def log_scrapy_info(settings: Settings) -> None:
         if name != "Scrapy"
     ]
     logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
-    log_reactor_info()
 
 
 def log_reactor_info() -> None:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index be067155e..118cb631b 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -271,6 +271,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
         self.assertEqual(runner.bootstrap_failed, True)
 
+    @defer.inlineCallbacks
     def test_crawler_runner_asyncio_enabled_true(self):
         if self.reactor_pytest == 'asyncio':
             CrawlerRunner(settings={
@@ -279,9 +280,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         else:
             msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
             with self.assertRaisesRegex(Exception, msg):
-                CrawlerRunner(settings={
+                runner = CrawlerRunner(settings={
                     "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
                 })
+                yield runner.crawl(NoRequestsSpider)
 
     @defer.inlineCallbacks
     # https://twistedmatrix.com/trac/ticket/9766
@@ -301,6 +303,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
                     runner = CrawlerProcess(settings={
                         "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
                     })
+                    yield runner.crawl(NoRequestsSpider)
 
     @defer.inlineCallbacks
     def test_crawler_process_asyncio_enabled_false(self):

From 041699b54cfa6cde9f886a98ff300e3276e2eaad Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Dec 2021 16:14:47 +0500
Subject: [PATCH 509/716] Remove tests that want to modify the test process
 reactor.

---
 tests/test_crawler.py | 27 ---------------------------
 1 file changed, 27 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 118cb631b..f445c181e 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -285,33 +285,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
                 })
                 yield runner.crawl(NoRequestsSpider)
 
-    @defer.inlineCallbacks
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
-    def test_crawler_process_asyncio_enabled_true(self):
-        with LogCapture(level=logging.DEBUG) as log:
-            if self.reactor_pytest == 'asyncio':
-                runner = CrawlerProcess(settings={
-                    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-                })
-                yield runner.crawl(NoRequestsSpider)
-                self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
-            else:
-                msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
-                with self.assertRaisesRegex(Exception, msg):
-                    runner = CrawlerProcess(settings={
-                        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-                    })
-                    yield runner.crawl(NoRequestsSpider)
-
-    @defer.inlineCallbacks
-    def test_crawler_process_asyncio_enabled_false(self):
-        runner = CrawlerProcess(settings={"TWISTED_REACTOR": None})
-        with LogCapture(level=logging.DEBUG) as log:
-            yield runner.crawl(NoRequestsSpider)
-            self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
-
 
 class ScriptRunnerMixin:
     def run_script(self, script_name, *script_args):

From ebcafdf4a9e0692bf301546b6d60465b3b2c4b06 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Dec 2021 16:35:26 +0500
Subject: [PATCH 510/716] Add tests for TWISTED_REACTOR in custom_settings.

---
 .../twisted_reactor_custom_settings.py        | 14 +++++++++++
 ...wisted_reactor_custom_settings_conflict.py | 22 +++++++++++++++++
 .../twisted_reactor_custom_settings_same.py   | 21 ++++++++++++++++
 tests/test_crawler.py                         | 24 +++++++++++++++++++
 4 files changed, 81 insertions(+)
 create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings.py
 create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
 create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings_same.py

diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.py
new file mode 100644
index 000000000..56304bd23
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py
@@ -0,0 +1,14 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class AsyncioReactorSpider(scrapy.Spider):
+    name = 'asyncio_reactor'
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+
+
+process = CrawlerProcess()
+process.crawl(AsyncioReactorSpider)
+process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
new file mode 100644
index 000000000..9a6c01d72
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
@@ -0,0 +1,22 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class PollReactorSpider(scrapy.Spider):
+    name = 'poll_reactor'
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor",
+    }
+
+
+class AsyncioReactorSpider(scrapy.Spider):
+    name = 'asyncio_reactor'
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+
+
+process = CrawlerProcess()
+process.crawl(PollReactorSpider)
+process.crawl(AsyncioReactorSpider)
+process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
new file mode 100644
index 000000000..1f5a44010
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
@@ -0,0 +1,21 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class AsyncioReactorSpider1(scrapy.Spider):
+    name = 'asyncio_reactor1'
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+
+class AsyncioReactorSpider2(scrapy.Spider):
+    name = 'asyncio_reactor2'
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+
+
+process = CrawlerProcess()
+process.crawl(AsyncioReactorSpider1)
+process.crawl(AsyncioReactorSpider2)
+process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index f445c181e..6d6763aec 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -361,6 +361,30 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
+    def test_reactor_asyncio_custom_settings(self):
+        log = self.run_script("twisted_reactor_custom_settings.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
+    def test_reactor_asyncio_custom_settings_same(self):
+        log = self.run_script("twisted_reactor_custom_settings_same.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
+    def test_reactor_asyncio_custom_settings_conflict(self):
+        log = self.run_script("twisted_reactor_custom_settings_conflict.py")
+        self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
+        self.assertIn("(twisted.internet.pollreactor.PollReactor) does not match the requested one", log)
+
     @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
     @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
     @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')

From 002513438204eea5062b5a1d75fb4f261880da4f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Dec 2021 16:45:17 +0500
Subject: [PATCH 511/716] Completely skip WindowsRunSpiderCommandTest outside
 Windows.

---
 tests/test_commands.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 75098a77a..efe9b0531 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -765,6 +765,7 @@ class MySpider(scrapy.Spider):
         self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
 
 
+@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
 class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
 
     spider_filename = 'myspider.pyw'
@@ -777,35 +778,27 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
         self.assertIn("start_requests", log)
         self.assertIn("badspider.pyw", log)
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_run_good_spider(self):
         super().test_run_good_spider()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_runspider(self):
         super().test_runspider()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_runspider_dnscache_disabled(self):
         super().test_runspider_dnscache_disabled()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_runspider_log_level(self):
         super().test_runspider_log_level()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_runspider_log_short_names(self):
         super().test_runspider_log_short_names()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_runspider_no_spider_found(self):
         super().test_runspider_no_spider_found()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_output(self):
         super().test_output()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_overwrite_output(self):
         super().test_overwrite_output()
 

From 9c4bfb48362f736fce81b71a6ca1fa0b3600231d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Dec 2021 17:17:36 +0500
Subject: [PATCH 512/716] Remove an unused import.

---
 tests/test_crawler.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 6d6763aec..d68c50026 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -7,7 +7,6 @@ import warnings
 from unittest import skipIf
 
 from pytest import raises, mark
-from testfixtures import LogCapture
 from twisted import version as twisted_version
 from twisted.internet import defer
 from twisted.python.versions import Version

From d4565318c7061c2ccd17fa5d5eabcacef8c34826 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Dec 2021 17:40:31 +0500
Subject: [PATCH 513/716] Fix a reactor test on Windows.

---
 .../twisted_reactor_custom_settings_conflict.py           | 8 ++++----
 tests/test_crawler.py                                     | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
index 9a6c01d72..3f219098c 100644
--- a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
@@ -2,10 +2,10 @@ import scrapy
 from scrapy.crawler import CrawlerProcess
 
 
-class PollReactorSpider(scrapy.Spider):
-    name = 'poll_reactor'
+class SelectReactorSpider(scrapy.Spider):
+    name = 'select_reactor'
     custom_settings = {
-        "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor",
+        "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
     }
 
 
@@ -17,6 +17,6 @@ class AsyncioReactorSpider(scrapy.Spider):
 
 
 process = CrawlerProcess()
-process.crawl(PollReactorSpider)
+process.crawl(SelectReactorSpider)
 process.crawl(AsyncioReactorSpider)
 process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index d68c50026..e7d5c8132 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -381,8 +381,8 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_reactor_asyncio_custom_settings_conflict(self):
         log = self.run_script("twisted_reactor_custom_settings_conflict.py")
-        self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
-        self.assertIn("(twisted.internet.pollreactor.PollReactor) does not match the requested one", log)
+        self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
+        self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log)
 
     @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
     @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')

From 940cc0776ff86f726e79c2ab2018f4b83a833936 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 24 Dec 2021 17:12:50 +0500
Subject: [PATCH 514/716] Add docs about TWISTED_REACTOR and other per-process
 settings.

---
 docs/topics/practices.rst | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 732eba587..bd0dd8ce0 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -102,6 +102,17 @@ reactor after ``MySpider`` has finished running.
     d.addBoth(lambda _: reactor.stop())
     reactor.run() # the script will block here until the crawling is finished
 
+.. note::
+    .. versionchanged:: VERSION
+
+    The Twisted reactor is now installed when
+    :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a
+    :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this,
+    :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now
+    honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions
+    they are silently ignored when set there and you need to set these settings
+    in some other way.
+
 .. seealso:: :doc:`twisted:core/howto/reactor-basics`
 
 .. _run-multiple-spiders:
@@ -193,6 +204,25 @@ Same example but running the spiders sequentially by chaining the deferreds:
     crawl()
     reactor.run() # the script will block here until the last crawl call is finished
 
+Different spiders can set different values for the same setting, but when they
+run in the same process it may be impossible, by design or because of some
+limitations, to use these different values. What happens in practice is
+different for different settings:
+
+* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value
+  (:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the
+  default one) cannot be read from the per-spider settings. These are applied
+  when the :class:`~scrapy.crawler.CrawlerRunner` or
+  :class:`~scrapy.crawler.CrawlerProcess` object is created.
+* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first
+  available value is used, and if a spider requests a different reactor an
+  exception will be raised. These are applied when the reactor is installed.
+* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the
+  ones used by the resolver (:setting:`DNSCACHE_ENABLED`,
+  :setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy)
+  the first available value is used. These are applied when the reactor is
+  started.
+
 .. seealso:: :ref:`run-from-script`.
 
 .. _distributed-crawls:

From a986792def6df1b2bbdf1bc996308d3afd8528c4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 24 Dec 2021 19:43:14 +0500
Subject: [PATCH 515/716] Add more docs for TWISTED_REACTOR.

---
 docs/topics/settings.rst | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 210c1def7..cff6d80cb 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1638,10 +1638,18 @@ which raises :exc:`Exception`, becomes::
 
 
 The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
-means that Scrapy will not attempt to install any specific reactor, and the
-default reactor defined by Twisted for the current platform will be used. This
-is to maintain backward compatibility and avoid possible problems caused by
-using a non-default reactor.
+means that Scrapy will install the default reactor defined by Twisted for the
+current platform will be used. This is to maintain backward compatibility and
+avoid possible problems caused by using a non-default reactor.
+
+.. note::
+    .. versionchanged:: VERSION
+
+    Previously this setting had no effect in a spider
+    :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but
+    if you :ref:`run several spiders in one process <run-multiple-spiders>`,
+    they must not have different values for this setting, because they will use
+    a single reactor instance.
 
 For additional information, see :doc:`core/howto/choosing-reactor`.
 

From a9dfd85ea6e983f255afc1b5b0f295fccddcbacb Mon Sep 17 00:00:00 2001
From: Burak Can Kahraman <brkcnkhrmn@gmail.com>
Date: Thu, 30 Dec 2021 15:48:53 +0300
Subject: [PATCH 516/716] Document coroutines for signals.

---
 docs/topics/coroutines.rst |  2 ++
 docs/topics/signals.rst    | 20 +++++++++-----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 2aef755c7..549552bd1 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -1,3 +1,5 @@
+.. _topics-coroutines:
+
 ==========
 Coroutines
 ==========
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 63ad3a9ad..328fb88d2 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -51,12 +51,12 @@ Deferred signal handlers
 ========================
 
 Some signals support returning :class:`~twisted.internet.defer.Deferred`
-objects from their handlers, allowing you to run asynchronous code that
-does not block Scrapy. If a signal handler returns a
-:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
-:class:`~twisted.internet.defer.Deferred` to fire.
+or :term:`awaitable objects <awaitable>` from their handlers, allowing
+you to run asynchronous code that does not block Scrapy. If a signal
+handler returns one of these objects, Scrapy waits for that asynchronous
+operation to finish.
 
-Let's take an example::
+Let's take an example using :ref:`coroutines <topics-coroutines>`::
 
     class SignalSpider(scrapy.Spider):
         name = 'signals'
@@ -68,17 +68,15 @@ Let's take an example::
             crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
             return spider
 
-        def item_scraped(self, item):
+        async def item_scraped(self, item):
             # Send the scraped item to the server
-            d = treq.post(
+            response = await treq.post(
                 'http://example.com/post',
                 json.dumps(item).encode('ascii'),
                 headers={b'Content-Type': [b'application/json']}
             )
 
-            # The next item will be scraped only after
-            # deferred (d) is fired
-            return d
+            return response
 
         def parse(self, response):
             for quote in response.css('div.quote'):
@@ -89,7 +87,7 @@ Let's take an example::
                 }
 
 See the :ref:`topics-signals-ref` below to know which signals support
-:class:`~twisted.internet.defer.Deferred`.
+:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects <awaitable>`.
 
 .. _topics-signals-ref:
 

From 7380888cad2c255e6f4a7efb6fe4cfd1240fb42c Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Thu, 30 Dec 2021 18:55:16 +0500
Subject: [PATCH 517/716] Fix a warning message. (#5359)

---
 scrapy/utils/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 24873f75d..24a6187b9 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None,
             message = (
                 'The -t command line option is deprecated in favor of '
                 'specifying the output format within the output URI. See the '
-                'documentation of the -o and -O options for more information.',
+                'documentation of the -o and -O options for more information.'
             )
             warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
             return {output[0]: {'format': output_format}}

From 64261d9e389737621caa85f320cf81ef2aef1faa Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 31 Dec 2021 15:45:59 +0500
Subject: [PATCH 518/716] Slight refactoring.

---
 scrapy/crawler.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 95cfb1bd1..a638254f1 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -71,17 +71,18 @@ class Crawler:
         lf_cls = load_object(self.settings['LOG_FORMATTER'])
         self.logformatter = lf_cls.from_crawler(self)
 
+        reactor_class = self.settings.get("TWISTED_REACTOR")
         if init_reactor:
             # this needs to be done after the spider settings are merged,
             # but before something imports twisted.internet.reactor
-            if self.settings.get("TWISTED_REACTOR"):
-                install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
+            if reactor_class:
+                install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
             else:
                 from twisted.internet import default
                 default.install()
             log_reactor_info()
-        if self.settings.get("TWISTED_REACTOR"):
-            verify_installed_reactor(self.settings["TWISTED_REACTOR"])
+        if reactor_class:
+            verify_installed_reactor(reactor_class)
 
         self.extensions = ExtensionManager.from_crawler(self)
 

From b81938684b55fca29e48faa766f2b6f6e3ab5d6a Mon Sep 17 00:00:00 2001
From: Andrey Oskin <oskin.a.v@gmail.com>
Date: Fri, 31 Dec 2021 21:49:18 +1100
Subject: [PATCH 519/716] Docs: correct process repetition start step (#5356)

The process repeats from step 3, the scheduler feeds request to the engine. Steps 1 and 2 are not parts of the loop as their incarnations steps 7 and 8 are parts of the loop.
---
 docs/topics/architecture.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 71d027c86..0c3a7ed88 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -67,7 +67,7 @@ this:
    the :ref:`Scheduler <component-scheduler>` and asks for possible next Requests
    to crawl.
 
-9. The process repeats (from step 1) until there are no more requests from the
+9. The process repeats (from step 3) until there are no more requests from the
    :ref:`Scheduler <component-scheduler>`.
 
 Components

From e4bdd1cb958b7d89b86ea66f0af1cec2d91a6d44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pawe=C5=82=20Miech?= <pawelmhm@users.noreply.github.com>
Date: Fri, 31 Dec 2021 11:57:12 +0100
Subject: [PATCH 520/716] downloader.webclient: make reactor import local
 (#5357)

---
 scrapy/core/downloader/webclient.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 915cb5fe3..06cb96489 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -3,7 +3,7 @@ from time import time
 from urllib.parse import urlparse, urlunparse, urldefrag
 
 from twisted.web.http import HTTPClient
-from twisted.internet import defer, reactor
+from twisted.internet import defer
 from twisted.internet.protocol import ClientFactory
 
 from scrapy.http import Headers
@@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
         p.followRedirect = self.followRedirect
         p.afterFoundGet = self.afterFoundGet
         if self.timeout:
+            from twisted.internet import reactor
             timeoutCall = reactor.callLater(self.timeout, p.timeout)
             self.deferred.addBoth(self._cancelTimeout, timeoutCall)
         return p

From 57dc58123b98e2026025cc87bdee474bf0656dcb Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Fri, 31 Dec 2021 17:15:08 +0500
Subject: [PATCH 521/716] Remove the experimental note about asyncio (#5332)

---
 docs/topics/asyncio.rst | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 28241ae24..402352721 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -10,11 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
 asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
 :mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
 
-.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet
-             recommended for production environments. Future Scrapy versions
-             may introduce related changes without a deprecation period or
-             warning.
-
 .. _install-asyncio:
 
 Installing the asyncio reactor

From a2763c608d33a9254034d650457b7efa7b434ec0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 31 Dec 2021 18:35:00 +0500
Subject: [PATCH 522/716] Remove unused
 MiddlewareManager._process_chain_both().

---
 scrapy/middleware.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index e8f60287a..2eb1d8609 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -9,7 +9,7 @@ from scrapy import Spider
 from scrapy.exceptions import NotConfigured
 from scrapy.settings import Settings
 from scrapy.utils.misc import create_instance, load_object
-from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
+from scrapy.utils.defer import process_parallel, process_chain
 
 logger = logging.getLogger(__name__)
 
@@ -72,11 +72,6 @@ class MiddlewareManager:
         methods = cast(Iterable[Callable], self.methods[methodname])
         return process_chain(methods, obj, *args)
 
-    def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred:
-        cb_methods = cast(Iterable[Callable], self.methods[cb_methodname])
-        eb_methods = cast(Iterable[Callable], self.methods[eb_methodname])
-        return process_chain_both(cb_methods, eb_methods, obj, *args)
-
     def open_spider(self, spider: Spider) -> Deferred:
         return self._process_parallel('open_spider', spider)
 

From 6eaceec735d551f5b777bc641ff8d85dbb3ba98c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 31 Dec 2021 20:14:24 +0500
Subject: [PATCH 523/716] Implement docs suggestions.

---
 docs/news.rst             | 22 ++++++++++++++++++++++
 docs/topics/practices.rst | 11 -----------
 docs/topics/settings.rst  | 13 ++-----------
 3 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 509366c17..2afe318f6 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1,3 +1,25 @@
+.. note::
+    .. versionchanged:: VERSION
+
+    The Twisted reactor is now installed when
+    :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a
+    :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this,
+    :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now
+    honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions
+    they are silently ignored when set there and you need to set these settings
+    in some other way.
+
+
+.. note::
+    .. versionchanged:: VERSION
+
+    Previously this setting had no effect in a spider
+    :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but
+    if you :ref:`run several spiders in one process <run-multiple-spiders>`,
+    they must not have different values for this setting, because they will use
+    a single reactor instance.
+
+
 .. _news:
 
 Release notes
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index bd0dd8ce0..1a9d56143 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -102,17 +102,6 @@ reactor after ``MySpider`` has finished running.
     d.addBoth(lambda _: reactor.stop())
     reactor.run() # the script will block here until the crawling is finished
 
-.. note::
-    .. versionchanged:: VERSION
-
-    The Twisted reactor is now installed when
-    :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a
-    :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this,
-    :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now
-    honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions
-    they are silently ignored when set there and you need to set these settings
-    in some other way.
-
 .. seealso:: :doc:`twisted:core/howto/reactor-basics`
 
 .. _run-multiple-spiders:
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index cff6d80cb..f6c95c502 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1639,17 +1639,8 @@ which raises :exc:`Exception`, becomes::
 
 The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
 means that Scrapy will install the default reactor defined by Twisted for the
-current platform will be used. This is to maintain backward compatibility and
-avoid possible problems caused by using a non-default reactor.
-
-.. note::
-    .. versionchanged:: VERSION
-
-    Previously this setting had no effect in a spider
-    :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but
-    if you :ref:`run several spiders in one process <run-multiple-spiders>`,
-    they must not have different values for this setting, because they will use
-    a single reactor instance.
+current platform. This is to maintain backward compatibility and avoid possible
+problems caused by using a non-default reactor.
 
 For additional information, see :doc:`core/howto/choosing-reactor`.
 

From c5ab58056c29c2c35b183572b0780acfbb15dfe8 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Sat, 1 Jan 2022 00:38:10 +0500
Subject: [PATCH 524/716] Set WindowsSelectorEventLoopPolicy on Windows (#5315)

---
 .github/workflows/tests-windows.yml           |  3 ++
 docs/topics/asyncio.rst                       | 28 +++++++++++++++++++
 scrapy/utils/reactor.py                       |  5 ++++
 .../CrawlerProcess/asyncio_enabled_reactor.py |  3 ++
 tests/test_commands.py                        | 11 +++-----
 tests/test_crawler.py                         | 16 -----------
 tests/test_downloader_handlers.py             | 16 +++++++++++
 tests/test_utils_asyncio.py                   |  7 +----
 8 files changed, 60 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 6fabf5cde..ab7385118 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -23,6 +23,9 @@ jobs:
         - python-version: "3.10"
           env:
             TOXENV: py
+        - python-version: "3.10"
+          env:
+            TOXENV: asyncio
 
     steps:
     - uses: actions/checkout@v2
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 402352721..8712d4268 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -36,6 +36,34 @@ use it instead of the default asyncio event loop.
 
 .. _asyncio-await-dfd:
 
+Windows-specific notes
+======================
+
+The Windows implementation of :mod:`asyncio` can use two event loop
+implementations: :class:`~asyncio.SelectorEventLoop` (default before Python
+3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop`
+(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the
+event loop class needs to be changed. Scrapy since VERSION does this
+automatically when you change the :setting:`TWISTED_REACTOR` setting or call
+:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor
+by other means or use an older Scrapy version you need to call the following
+code before installing the reactor::
+
+    import asyncio
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+You can put this in the same function that installs the reactor, if you do that
+yourself, or in some code that runs before the reactor is installed, e.g.
+``settings.py``.
+
+.. note:: Other libraries you use may require
+          :class:`~asyncio.ProactorEventLoop`, e.g. because it supports
+          subprocesses (this is the case with `playwright`_), so you cannot use
+          them together with Scrapy on Windows (but you should be able to use
+          them on WSL or native Linux).
+
+.. _playwright: https://github.com/microsoft/playwright-python
+
 Awaiting on Deferreds
 =====================
 
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 6723d9b37..96395543c 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,4 +1,5 @@
 import asyncio
+import sys
 from contextlib import suppress
 
 from twisted.internet import asyncioreactor, error
@@ -57,6 +58,10 @@ def install_reactor(reactor_path, event_loop_path=None):
     reactor_class = load_object(reactor_path)
     if reactor_class is asyncioreactor.AsyncioSelectorReactor:
         with suppress(error.ReactorAlreadyInstalledError):
+            if sys.version_info >= (3, 8) and sys.platform == "win32":
+                policy = asyncio.get_event_loop_policy()
+                if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy):
+                    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
             if event_loop_path is not None:
                 event_loop_class = load_object(event_loop_path)
                 event_loop = event_loop_class()
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py
index 8568bd8b8..f2a93074b 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@@ -1,6 +1,9 @@
 import asyncio
+import sys
 
 from twisted.internet import asyncioreactor
+if sys.version_info >= (3, 8) and sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncioreactor.install(asyncio.get_event_loop())
 
 import scrapy
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 75098a77a..81d1a1cab 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -674,9 +674,6 @@ class MySpider(scrapy.Spider):
         self.assertIn("start_requests", log)
         self.assertIn("badspider.py", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_true(self):
         log = self.get_log(self.debug_log_spider, args=[
             '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
@@ -699,15 +696,15 @@ class MySpider(scrapy.Spider):
         ])
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_custom_asyncio_loop_enabled_false(self):
         log = self.get_log(self.debug_log_spider, args=[
             '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
         ])
         import asyncio
-        loop = asyncio.new_event_loop()
+        if sys.platform != 'win32':
+            loop = asyncio.new_event_loop()
+        else:
+            loop = asyncio.SelectorEventLoop()
         self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log)
 
     def test_output(self):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index be067155e..7bc4fba40 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -4,7 +4,6 @@ import platform
 import subprocess
 import sys
 import warnings
-from unittest import skipIf
 
 from pytest import raises, mark
 from testfixtures import LogCapture
@@ -284,9 +283,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
                 })
 
     @defer.inlineCallbacks
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_crawler_process_asyncio_enabled_true(self):
         with LogCapture(level=logging.DEBUG) as log:
             if self.reactor_pytest == 'asyncio':
@@ -328,17 +324,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn('Spider closed (finished)', log)
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_no_reactor(self):
         log = self.run_script('asyncio_enabled_no_reactor.py')
         self.assertIn('Spider closed (finished)', log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_asyncio_enabled_reactor(self):
         log = self.run_script('asyncio_enabled_reactor.py')
         self.assertIn('Spider closed (finished)', log)
@@ -377,9 +367,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_reactor_asyncio(self):
         log = self.run_script("twisted_reactor_asyncio.py")
         self.assertIn("Spider closed (finished)", log)
@@ -404,9 +391,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
         self.assertIn("async pipeline opened!", log)
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_default_loop_asyncio_deferred_signal(self):
         log = self.run_script("asyncio_deferred_signal.py")
         self.assertIn("Spider closed (finished)", log)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 9c11820e5..a1ea4c679 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,6 +1,7 @@
 import contextlib
 import os
 import shutil
+import sys
 import tempfile
 from typing import Optional, Type
 from unittest import mock
@@ -287,6 +288,12 @@ class HttpTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_download_from_spider_nodata_rcvd(self):
+        if self.reactor_pytest == "asyncio" and sys.platform == "win32":
+            # https://twistedmatrix.com/trac/ticket/10279
+            raise unittest.SkipTest(
+                "This test produces DirtyReactorAggregateError on Windows with asyncio"
+            )
+
         # client connects but no data is received
         spider = Spider('foo')
         meta = {'download_timeout': 0.5}
@@ -296,6 +303,11 @@ class HttpTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_download_from_spider_server_hangs(self):
+        if self.reactor_pytest == "asyncio" and sys.platform == "win32":
+            # https://twistedmatrix.com/trac/ticket/10279
+            raise unittest.SkipTest(
+                "This test produces DirtyReactorAggregateError on Windows with asyncio"
+            )
         # client connects, server send headers and some body bytes but hangs
         spider = Spider('foo')
         meta = {'download_timeout': 0.5}
@@ -1055,6 +1067,10 @@ class BaseFTPTestCase(unittest.TestCase):
 class FTPTestCase(BaseFTPTestCase):
 
     def test_invalid_credentials(self):
+        if self.reactor_pytest == "asyncio" and sys.platform == "win32":
+            raise unittest.SkipTest(
+                "This test produces DirtyReactorAggregateError on Windows with asyncio"
+            )
         from twisted.protocols.ftp import ConnectionLost
 
         meta = dict(self.req_meta)
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index a2114bd18..295323e4d 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -1,6 +1,4 @@
-import platform
-import sys
-from unittest import skipIf, TestCase
+from unittest import TestCase
 
 from pytest import mark
 
@@ -14,9 +12,6 @@ class AsyncioTest(TestCase):
         # the result should depend only on the pytest --reactor argument
         self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
 
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_install_asyncio_reactor(self):
         # this should do nothing
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

From f789547551ae0eb79f41c9de44525bf597a0ffa5 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 11 Jan 2022 18:28:32 +0500
Subject: [PATCH 525/716] Implement spider middleware iterable
 upgrade/downgrade.

---
 docs/topics/coroutines.rst                  | 104 ++++---
 docs/topics/spider-middleware.rst           |   9 +
 scrapy/core/scraper.py                      |   3 +-
 scrapy/core/spidermw.py                     | 135 +++++++--
 scrapy/middleware.py                        |   7 +-
 scrapy/spidermiddlewares/depth.py           |  71 ++---
 scrapy/spidermiddlewares/offsite.py         |  40 +--
 scrapy/spidermiddlewares/referer.py         |  22 +-
 scrapy/spidermiddlewares/urllength.py       |  33 +--
 scrapy/utils/asyncgen.py                    |  47 +--
 scrapy/utils/python.py                      |   4 +-
 tests/test_spidermiddleware.py              | 302 +++++++++++++++++---
 tests/test_spidermiddleware_output_chain.py |  17 ++
 tests/test_utils_asyncgen.py                |  52 +---
 14 files changed, 553 insertions(+), 293 deletions(-)

diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index c514fc00a..073b6bd9a 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -35,11 +35,10 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 -   :ref:`Signal handlers that support deferreds <signal-deferred>`.
 
 -   The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
-    method of :ref:`spider middlewares <custom-spider-middleware>`.
+    method of :ref:`spider middlewares <custom-spider-middleware>`. See
+    :ref:`async-spider-middlewares`.
 
     .. versionadded:: VERSION
-    .. note:: This method needs to be an async generator, not just a coroutine that
-              returns an iterable.
 
 Usage
 =====
@@ -106,8 +105,8 @@ Common use cases for asynchronous code include:
 * storing data in databases (in pipelines and middlewares);
 * delaying the spider initialization until some external event (in the
   :signal:`spider_opened` handler);
-* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see
-  :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
+* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download`
+  (see :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
 
 .. _aio-libs: https://github.com/aio-libs
 
@@ -119,59 +118,72 @@ Asynchronous spider middlewares
 .. versionadded:: VERSION
 .. note:: This currently applies to
           :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`.
+          In the future it will also apply to
+          :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests`.
 
 Middleware methods discussed here can take and return async iterables. They can
 return the same type of iterable or they can take a normal one and return an
 async one. If such method needs to return an async iterable it must be an async
 generator, not just a coroutine that returns an iterable.
 
-.. autofunction:: scrapy.utils.asyncgen.as_async_generator
+As the result of a middleware method is passed to the same method of the next
+middleware, it needs to be adapted if the second method expects a different
+type. Scrapy will do this transparently:
 
-In the simplest form that supports both sync and async input it can be written
-like this::
+* A normal iterable is wrapped into an async one which shouldn't cause any side
+  effects.
+* An async iterable is downgraded to a normal one by waiting until all results
+  are available and wrapping them in a normal iterable. This is problematic
+  because it pauses the normal middleware processing for this iterable and
+  because all results can be skipped if exceptions are raised during
+  processing. This case emits a warning and will be deprecated and then removed
+  in a later Scrapy version.
+* Async iterables returned from
+  :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception`
+  won't be downgraded, an exception will be raised if that is needed.
 
-    from scrapy.utils.asyncgen import as_async_generator
+As downgrading is undesirable, here is the proposed way to avoid it. If all
+middlewares, including 3rd-party ones, support async iterables as input, no
+downgrading will happen. But removing normal iterable support (making the
+method a coroutine) from a middleware published as a separate project or used
+internally in projects for older Scrapy versions breaks backwards
+compatibility. So, as an interim measure (it will be deprecated and then
+removed in a later Scrapy version), a middleware can provide both sync and
+async methods in the following form::
 
-    class ProcessSpiderOutputAsyncGenMiddleware:
-        async def process_spider_output(self, response, result, spider):
-            async for r in as_async_generator(result):
+    class UniversalSpiderMiddleware:
+        def process_spider_output(self, response, result, spider):
+            for r in result:
                 # ... do something with r
                 yield r
 
-If the middleware input (the callback result for ``process_spider_output``) is
-an async iterable, all middlewares that process it must support it. The
-built-in ones do, but the ones in your project and 3rd-party ones will need to
-be updated to support it, as the code that expects a normal iterable will break
-on an async one. If these middlewares receive an async iterable, they must
-return one as well. On the other hand, if they receive a normal iterable, they
-shouldn't break and ideally should return a normal iterable too. There can be
-several possible implementations of this.
+        async def process_spider_output_async(self, response, result, spider):
+            async for r in result:
+                # ... do something with r
+                yield r
 
-The simplest one, always converting normal iterables to async ones, is provided
-above. Because a result of a middleware method is passed to the same method of
-the next middleware, it's only possible to mix middlewares with synchronous and
-asynchronous implementations of the same method if all synchronous ones are
-called first (which isn't always possible).
+In this case normal and async iterables will be passed to the respective
+methods without any wrapping or downgrading, and in older versions of Scrapy
+the coroutine method will just be ignored. When the backwards compatibility is
+no longer needed the non-coroutine method can be dropped and the coroutine one
+renamed to the normal name. It may be possible to extract common code from both
+methods to reduce code duplication, as in the simplest case the only difference
+between them will be ``for`` vs ``async for``.
 
-Another option is to make separate methods for normal and async iterables and
-choose one at run time::
+So, to recap:
 
-    from inspect import isasyncgen
-
-    class ProcessSpiderOutputAsyncGenMiddleware:
-        def _normal_process_spider_output(self, response, result, spider):
-            # ... do something with normal result
-
-        async def _async_process_spider_output(self, response, result, spider):
-            # ... do the same with async result
-
-        def process_spider_output(self, response, result, spider):
-            if isasyncgen(result):
-                return self._async_process_spider_output(self, response, result, spider)
-            else:
-                return self._normal_process_spider_output(self, response, result, spider)
-
-If you are writing a middleware that you intend to publish or to use in many
-projects, this is likely the best way to implement it. It may be possible to
-extract common code from both methods to reduce code duplication, as in the
-simplest case the only difference between them will be ``for`` vs ``async for``.
+* If you don't intend to use async callbacks or middlewares containing async
+  code in your project, nothing should change for you yet. At some point in the
+  future some of the 3rd-party middlewares you use may drop backwards
+  compatibility, which shouldn't lead to immediate problems but may be a sign
+  to start converting your code to ``async def`` too.
+* If you maintain a middleware that can be used with projects you can't control
+  (e.g. one you published for other people to use, or one that needs to support
+  some old project that can't be modernized), we recommend adding a
+  ``process_spider_output_async`` method so that the amount of unnecessary
+  iterable conversions is reduced but no compatibility is broken.
+* If you use async callbacks, try to make sure all middlewares support them.
+  Note that you can modernize 3rd-party middlewares by subclassing them.
+* If you want to write and publish a middleware that requires async code, you
+  should write in the docs that the minimum support Scrapy version is VERSION
+  (maybe even check this at the run time, using :attr:`scrapy.__version__`).
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index f8d4a356f..edfc2e4bb 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -123,6 +123,15 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         :param spider: the spider whose result is being processed
         :type spider: :class:`~scrapy.Spider` object
 
+    .. method:: process_spider_output_async(response, result, spider)
+
+        .. versionadded:: VERSION
+
+        If exists, this methid will be called instead of
+        :meth:`process_spider_output` when ``result`` is an async iterable.
+        If this method exists, it must be a coroutine while
+        :meth:`process_spider_output` must not be a coroutine.
+
     .. method:: process_spider_exception(response, exception, spider)
 
         This method is called when a spider or :meth:`process_spider_output`
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index e64cfae0a..e1fdd8d13 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -192,7 +192,8 @@ class Scraper:
             spider=spider
         )
 
-    def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred:
+    def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request,
+                             response: Response, spider: Spider) -> Deferred:
         if not result:
             return defer_succeed(None)
         it: Union[Generator, AsyncGenerator]
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 009ece06d..6075670b0 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,22 +3,27 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
+import logging
+from inspect import isasyncgenfunction
 from itertools import islice
-from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union, cast
+from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast
 
-from twisted.internet.defer import Deferred
+from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.python.failure import Failure
 
 from scrapy import Request, Spider
 from scrapy.exceptions import _InvalidOutput
 from scrapy.http import Response
 from scrapy.middleware import MiddlewareManager
-from scrapy.utils.asyncgen import _process_iterable_universal
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.conf import build_component_list
-from scrapy.utils.defer import mustbe_deferred
+from scrapy.utils.defer import mustbe_deferred, deferred_from_coro, deferred_f_from_coro_f, maybe_deferred_to_future
 from scrapy.utils.python import MutableAsyncChain, MutableChain
 
 
+logger = logging.getLogger(__name__)
+
+
 ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
 
 
@@ -30,6 +35,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
 
+    def __init__(self, *middlewares):
+        super().__init__(*middlewares)
+        self.downgrade_warning_done = False
+
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
         return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
@@ -40,7 +49,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             self.methods['process_spider_input'].append(mw.process_spider_input)
         if hasattr(mw, 'process_start_requests'):
             self.methods['process_start_requests'].appendleft(mw.process_start_requests)
-        process_spider_output = getattr(mw, 'process_spider_output', None)
+        process_spider_output = self._get_async_method_pair(mw, 'process_spider_output')
         self.methods['process_spider_output'].appendleft(process_spider_output)
         process_spider_exception = getattr(mw, 'process_spider_exception', None)
         self.methods['process_spider_exception'].appendleft(process_spider_exception)
@@ -64,8 +73,19 @@ class SpiderMiddlewareManager(MiddlewareManager):
     def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable],
                            exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain]
                            ) -> Union[Generator, AsyncGenerator]:
-        @_process_iterable_universal
-        async def _evaluate_async_iterable(iterable):
+
+        def process_sync(iterable: Iterable):
+            try:
+                for r in iterable:
+                    yield r
+            except Exception as ex:
+                exception_result = self._process_spider_exception(response, spider, Failure(ex),
+                                                                  exception_processor_index)
+                if isinstance(exception_result, Failure):
+                    raise
+                recover_to.extend(exception_result)
+
+        async def process_async(iterable: AsyncIterable):
             try:
                 async for r in iterable:
                     yield r
@@ -75,7 +95,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 if isinstance(exception_result, Failure):
                     raise
                 recover_to.extend(exception_result)
-        return _evaluate_async_iterable(iterable)
+
+        if isinstance(iterable, AsyncIterable):
+            return process_async(iterable)
+        return process_sync(iterable)
 
     def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure,
                                   start_index: int = 0) -> Union[Failure, MutableChain]:
@@ -87,11 +110,22 @@ class SpiderMiddlewareManager(MiddlewareManager):
         for method_index, method in enumerate(method_list, start=start_index):
             if method is None:
                 continue
+            method = cast(Callable, method)
             result = method(response=response, exception=exception, spider=spider)
             if _isiterable(result):
                 # stop exception handling by handing control over to the
                 # process_spider_output chain if an iterable has been returned
-                return self._process_spider_output(response, spider, result, method_index + 1)
+                dfd: Deferred = self._process_spider_output(response, spider, result, method_index + 1)
+                # _process_spider_output() returns a Deferred only because of downgrading so this can be
+                # simplified when downgrading is removed.
+                if dfd.called:
+                    # the result is available immediately if _process_spider_output didn't do downgrading
+                    return dfd.result
+                else:
+                    # we forbid waiting here because otherwise we would need to return a deferred from
+                    # _process_spider_exception too, which complicates the architecture
+                    msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
+                    raise _InvalidOutput(msg)
             elif result is None:
                 continue
             else:
@@ -100,9 +134,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 raise _InvalidOutput(msg)
         return _failure
 
+    # This method cannot be made async def, as _process_spider_exception relies on the Deferred result
+    # being available immediately which doesn't work when it's a wrapped coroutine.
+    # It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed.
+    @inlineCallbacks
     def _process_spider_output(self, response: Response, spider: Spider,
                                result: Union[Iterable, AsyncIterable], start_index: int = 0
-                               ) -> Union[MutableChain, MutableAsyncChain]:
+                               ) -> Deferred:
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
         recovered: Union[MutableChain, MutableAsyncChain]
@@ -112,11 +150,43 @@ class SpiderMiddlewareManager(MiddlewareManager):
         else:
             recovered = MutableChain()
 
+        # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async.
+        # 1. def foo. Sync iterables are passed as is, async ones are downgraded.
+        # 2. async def foo. Sync iterables are upgraded, async ones are passed as is.
+        # 3. def foo + async def foo_async. Iterables are passed to the respective method.
+        # Storing methods and method tuples in the same list is weird but we should be able to roll this back
+        # when we drop this compatibility feature.
+
         method_list = islice(self.methods['process_spider_output'], start_index, None)
-        for method_index, method in enumerate(method_list, start=start_index):
-            if method is None:
+        for method_index, method_pair in enumerate(method_list, start=start_index):
+            if method_pair is None:
                 continue
+            need_upgrade = need_downgrade = False
+            if isinstance(method_pair, tuple):
+                # This tuple handling is only needed until _async compatibility methods are removed.
+                method_sync, method_async = method_pair
+                method = method_async if last_result_is_async else method_sync
+            else:
+                method = method_pair
+                if not last_result_is_async and isasyncgenfunction(method):
+                    need_upgrade = True
+                elif last_result_is_async and not isasyncgenfunction(method):
+                    need_downgrade = True
             try:
+                if need_upgrade:
+                    # Iterable -> AsyncIterable
+                    result = as_async_generator(result)
+                elif need_downgrade:
+                    if not self.downgrade_warning_done:
+                        logger.warning(f"Async iterable passed to {method.__qualname__} "
+                                       f"was downgraded to a non-async one")
+                        self.downgrade_warning_done = True
+                    assert isinstance(result, AsyncIterable)
+                    # AsyncIterable -> Iterable
+                    result = yield deferred_from_coro(collect_asyncgen(result))
+                    if isinstance(recovered, AsyncIterable):
+                        recovered_collected = yield deferred_from_coro(collect_asyncgen(recovered))
+                        recovered = MutableChain(recovered_collected)
                 # might fail directly if the output value is not a generator
                 result = method(response=response, result=result, spider=spider)
             except Exception as ex:
@@ -130,8 +200,6 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 msg = (f"Middleware {method.__qualname__} must return an "
                        f"iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
-            if last_result_is_async and isinstance(result, Iterable):
-                raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")
             last_result_is_async = isinstance(result, AsyncIterable)
 
         if last_result_is_async:
@@ -139,31 +207,58 @@ class SpiderMiddlewareManager(MiddlewareManager):
         else:
             return MutableChain(result, recovered)  # type: ignore[arg-type]
 
-    def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
-                                 ) -> Union[MutableChain, MutableAsyncChain]:
+    async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
+                                       ) -> Union[MutableChain, MutableAsyncChain]:
         recovered: Union[MutableChain, MutableAsyncChain]
         if isinstance(result, AsyncIterable):
             recovered = MutableAsyncChain()
         else:
             recovered = MutableChain()
         result = self._evaluate_iterable(response, spider, result, 0, recovered)
-        result = self._process_spider_output(response, spider, result)
+        result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result))
         if isinstance(result, AsyncIterable):
             return MutableAsyncChain(result, recovered)
         else:
+            if isinstance(recovered, AsyncIterable):
+                recovered_collected = await collect_asyncgen(recovered)
+                recovered = MutableChain(recovered_collected)
             return MutableChain(result, recovered)  # type: ignore[arg-type]
 
     def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
                         spider: Spider) -> Deferred:
-        def process_callback_output(result: Union[Iterable, AsyncIterable]) -> Union[MutableChain, MutableAsyncChain]:
-            return self._process_callback_output(response, spider, result)
+        async def process_callback_output(result: Union[Iterable, AsyncIterable]
+                                          ) -> Union[MutableChain, MutableAsyncChain]:
+            return await self._process_callback_output(response, spider, result)
 
         def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
             return self._process_spider_exception(response, spider, _failure)
 
         dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
-        dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
+        dfd.addCallbacks(callback=deferred_f_from_coro_f(process_callback_output), errback=process_spider_exception)
         return dfd
 
     def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
         return self._process_chain('process_start_requests', start_requests, spider)
+
+    # This method is only needed until _async compatibility methods are removed.
+    @staticmethod
+    def _get_async_method_pair(mw: Any, methodname: str) -> Union[None, Callable, Tuple[Callable, Callable]]:
+        normal_method = getattr(mw, methodname, None)
+        methodname_async = methodname + "_async"
+        async_method = getattr(mw, methodname_async, None)
+        if not async_method:
+            return normal_method
+        if not normal_method:
+            logger.error(f"Middleware {mw.__qualname__} has {methodname_async} "
+                         f"without {methodname}, skipping this method.")
+            return None
+        if not isasyncgenfunction(async_method):
+            logger.error(f"{async_method.__qualname__} is not "
+                         f"an async generator function, skipping this method.")
+            return normal_method
+        if isasyncgenfunction(normal_method):
+            logger.error(f"{normal_method.__qualname__} is an async "
+                         f"generator function while {methodname_async} exists, "
+                         f"skipping both methods.")
+            return None
+        return normal_method, async_method
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 2eb1d8609..8d7e5a602 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,7 +1,7 @@
 import logging
 import pprint
 from collections import defaultdict, deque
-from typing import Callable, Deque, Dict, Optional, cast, Iterable
+from typing import Callable, Deque, Dict, Iterable, Tuple, Union, cast
 
 from twisted.internet.defer import Deferred
 
@@ -21,8 +21,9 @@ class MiddlewareManager:
 
     def __init__(self, *middlewares):
         self.middlewares = middlewares
-        # Optional because process_spider_output and process_spider_exception can be None
-        self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque)
+        # Only process_spider_output and process_spider_exception can be None.
+        # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed.
+        self.methods: Dict[str, Deque[Union[None, Callable, Tuple[Callable, Callable]]]] = defaultdict(deque)
         for mw in middlewares:
             self._add_middleware(mw)
 
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 973404b2b..29634c3ad 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -7,7 +7,6 @@ See documentation in docs/topics/spider-middleware.rst
 import logging
 
 from scrapy.http import Request
-from scrapy.utils.asyncgen import _process_iterable_universal
 
 logger = logging.getLogger(__name__)
 
@@ -29,36 +28,42 @@ class DepthMiddleware:
         return cls(maxdepth, crawler.stats, verbose, prio)
 
     def process_spider_output(self, response, result, spider):
-        def _filter(request):
-            if isinstance(request, Request):
-                depth = response.meta['depth'] + 1
-                request.meta['depth'] = depth
-                if self.prio:
-                    request.priority -= depth * self.prio
-                if self.maxdepth and depth > self.maxdepth:
-                    logger.debug(
-                        "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
-                        {'maxdepth': self.maxdepth, 'requrl': request.url},
-                        extra={'spider': spider}
-                    )
-                    return False
-                else:
-                    if self.verbose_stats:
-                        self.stats.inc_value(f'request_depth_count/{depth}',
-                                             spider=spider)
-                    self.stats.max_value('request_depth_max', depth,
-                                         spider=spider)
+        # base case (depth=0)
+        if 'depth' not in response.meta:
+            response.meta['depth'] = 0
+            if self.verbose_stats:
+                self.stats.inc_value('request_depth_count/0', spider=spider)
+
+        return (r for r in result or () if self._filter(r, response, spider))
+
+    async def process_spider_output_async(self, response, result, spider):
+        # base case (depth=0)
+        if 'depth' not in response.meta:
+            response.meta['depth'] = 0
+            if self.verbose_stats:
+                self.stats.inc_value('request_depth_count/0', spider=spider)
+
+        async for r in result or ():
+            if self._filter(r, response, spider):
+                yield r
+
+    def _filter(self, request, response, spider):
+        if not isinstance(request, Request):
             return True
-
-        @_process_iterable_universal
-        async def process(result):
-            # base case (depth=0)
-            if 'depth' not in response.meta:
-                response.meta['depth'] = 0
-                if self.verbose_stats:
-                    self.stats.inc_value('request_depth_count/0', spider=spider)
-
-            async for r in result or ():
-                if _filter(r):
-                    yield r
-        return process(result)
+        depth = response.meta['depth'] + 1
+        request.meta['depth'] = depth
+        if self.prio:
+            request.priority -= depth * self.prio
+        if self.maxdepth and depth > self.maxdepth:
+            logger.debug(
+                "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
+                {'maxdepth': self.maxdepth, 'requrl': request.url},
+                extra={'spider': spider}
+            )
+            return False
+        if self.verbose_stats:
+            self.stats.inc_value(f'request_depth_count/{depth}',
+                                 spider=spider)
+        self.stats.max_value('request_depth_max', depth,
+                             spider=spider)
+        return True
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 074ec7a4e..448bc1367 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -9,7 +9,6 @@ import warnings
 
 from scrapy import signals
 from scrapy.http import Request
-from scrapy.utils.asyncgen import _process_iterable_universal
 from scrapy.utils.httpobj import urlparse_cached
 
 logger = logging.getLogger(__name__)
@@ -27,24 +26,27 @@ class OffsiteMiddleware:
         return o
 
     def process_spider_output(self, response, result, spider):
-        @_process_iterable_universal
-        async def process(result):
-            async for x in result:
-                if isinstance(x, Request):
-                    if x.dont_filter or self.should_follow(x, spider):
-                        yield x
-                    else:
-                        domain = urlparse_cached(x).hostname
-                        if domain and domain not in self.domains_seen:
-                            self.domains_seen.add(domain)
-                            logger.debug(
-                                "Filtered offsite request to %(domain)r: %(request)s",
-                                {'domain': domain, 'request': x}, extra={'spider': spider})
-                            self.stats.inc_value('offsite/domains', spider=spider)
-                        self.stats.inc_value('offsite/filtered', spider=spider)
-                else:
-                    yield x
-        return process(result)
+        return (r for r in result or () if self._filter(r, spider))
+
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result or ():
+            if self._filter(r, spider):
+                yield r
+
+    def _filter(self, request, spider) -> bool:
+        if not isinstance(request, Request):
+            return True
+        if request.dont_filter or self.should_follow(request, spider):
+            return True
+        domain = urlparse_cached(request).hostname
+        if domain and domain not in self.domains_seen:
+            self.domains_seen.add(domain)
+            logger.debug(
+                "Filtered offsite request to %(domain)r: %(request)s",
+                {'domain': domain, 'request': request}, extra={'spider': spider})
+            self.stats.inc_value('offsite/domains', spider=spider)
+        self.stats.inc_value('offsite/filtered', spider=spider)
+        return False
 
     def should_follow(self, request, spider):
         regex = self.host_regex
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index e0a22592f..8027beb92 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -11,7 +11,6 @@ from w3lib.url import safe_url_string
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Request, Response
-from scrapy.utils.asyncgen import _process_iterable_universal
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_unicode
 from scrapy.utils.url import strip_url
@@ -334,19 +333,18 @@ class RefererMiddleware:
         return cls() if cls else self.default_policy()
 
     def process_spider_output(self, response, result, spider):
-        def _set_referer(r):
-            if isinstance(r, Request):
-                referrer = self.policy(response, r).referrer(response.url, r.url)
-                if referrer is not None:
-                    r.headers.setdefault('Referer', referrer)
-            return r
+        return (self._set_referer(r, response) for r in result or ())
 
-        @_process_iterable_universal
-        async def process(result):
-            async for r in result or ():
-                yield _set_referer(r)
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result or ():
+            yield self._set_referer(r, response)
 
-        return process(result)
+    def _set_referer(self, r, response):
+        if isinstance(r, Request):
+            referrer = self.policy(response, r).referrer(response.url, r.url)
+            if referrer is not None:
+                r.headers.setdefault('Referer', referrer)
+        return r
 
     def request_scheduled(self, request, spider):
         # check redirected request to patch "Referer" header if necessary
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index 63b1d36bd..7ad64d2af 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -8,7 +8,6 @@ import logging
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
-from scrapy.utils.asyncgen import _process_iterable_universal
 
 logger = logging.getLogger(__name__)
 
@@ -26,22 +25,20 @@ class UrlLengthMiddleware:
         return cls(maxlength)
 
     def process_spider_output(self, response, result, spider):
-        def _filter(request):
-            if isinstance(request, Request) and len(request.url) > self.maxlength:
-                logger.info(
-                    "Ignoring link (url length > %(maxlength)d): %(url)s ",
-                    {'maxlength': self.maxlength, 'url': request.url},
-                    extra={'spider': spider}
-                )
-                spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
-                return False
-            else:
-                return True
+        return (r for r in result or () if self._filter(r, spider))
 
-        @_process_iterable_universal
-        async def process(result):
-            async for r in result or ():
-                if _filter(r):
-                    yield r
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result or ():
+            if self._filter(r, spider):
+                yield r
 
-        return process(result)
+    def _filter(self, request, spider):
+        if isinstance(request, Request) and len(request.url) > self.maxlength:
+            logger.info(
+                "Ignoring link (url length > %(maxlength)d): %(url)s ",
+                {'maxlength': self.maxlength, 'url': request.url},
+                extra={'spider': spider}
+            )
+            spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
+            return False
+        return True
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index ae9a79989..9f794de92 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,6 +1,4 @@
-import functools
-import inspect
-from typing import AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union
+from typing import AsyncGenerator, AsyncIterable, Iterable, Union
 
 
 async def collect_asyncgen(result: AsyncIterable):
@@ -18,46 +16,3 @@ async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerat
     else:
         for r in it:
             yield r
-
-
-# https://stackoverflow.com/a/66170760/113586
-def _process_iterable_universal(process_async: Callable):
-    """ Takes a function that takes an async iterable, args and kwargs. Returns
-    a function that takes any iterable, args and kwargs.
-
-    Requires that process_async only awaits on the iterable and synchronous functions,
-    so it's better to use this only in the Scrapy code itself.
-    """
-
-    # If this stops working, all internal uses can be just replaced with manually-written
-    # process_sync functions.
-
-    def process_sync(iterable: Iterable, *args, **kwargs) -> Generator:
-        agen = process_async(as_async_generator(iterable), *args, **kwargs)
-        if not inspect.isasyncgen(agen):
-            raise ValueError(f"process_async returned wrong type {type(agen)}")
-        sent = None
-        while True:
-            try:
-                gen = agen.asend(sent)
-                gen.send(None)
-            except StopIteration as e:
-                sent = yield e.value
-            except StopAsyncIteration:
-                return
-            else:
-                gen.throw(RuntimeError,
-                          f"Synchronously-called function '{process_async.__name__}' has blocked, "
-                          f"you can't use {_process_iterable_universal.__name__} with it.")
-
-    @functools.wraps(process_async)
-    def process(iterable: Union[Iterable, AsyncIterable], *args, **kwargs) -> Union[Generator, AsyncGenerator]:
-        if isinstance(iterable, AsyncIterable):
-            # call process_async directly
-            return process_async(iterable, *args, **kwargs)
-        if isinstance(iterable, Iterable):
-            # convert process_async to process_sync
-            return process_sync(iterable, *args, **kwargs)
-        raise TypeError(f"Wrong iterable type {type(iterable)}")
-
-    return process
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index d086347bc..11c089ac2 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -10,7 +10,7 @@ import warnings
 import weakref
 from functools import partial, wraps
 from itertools import chain
-from typing import AsyncIterable, Iterable, Union
+from typing import AsyncGenerator, AsyncIterable, Iterable, Union
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.asyncgen import as_async_generator
@@ -359,7 +359,7 @@ class MutableChain(Iterable):
         return self.__next__()
 
 
-async def _async_chain(*iterables: Union[Iterable, AsyncIterable]):
+async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
     for it in iterables:
         async for o in as_async_generator(it):
             yield o
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index b0ca2f62e..f9f2b6642 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,6 +1,8 @@
 import collections.abc
+from typing import Optional
 from unittest import mock
 
+from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
@@ -8,8 +10,8 @@ from twisted.python.failure import Failure
 from scrapy.spiders import Spider
 from scrapy.http import Request, Response
 from scrapy.exceptions import _InvalidOutput
-from scrapy.utils.asyncgen import _process_iterable_universal, as_async_generator, collect_asyncgen
-from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.asyncgen import collect_asyncgen
+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
 from scrapy.core.spidermw import SpiderMiddlewareManager
 
@@ -115,33 +117,40 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
 
     RESULT_COUNT = 3  # to simplify checks, let everything return 3 objects
 
+    @staticmethod
+    def _construct_mw_setting(*mw_classes, start_index: Optional[int] = None):
+        if start_index is None:
+            start_index = 10
+        return {i: c for c, i in enumerate(mw_classes, start=start_index)}
+
     @defer.inlineCallbacks
-    def _get_middleware_result(self, *mw_classes):
-        for mw_cls in mw_classes:
-            self.mwman._add_middleware(mw_cls())
+    def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
+        setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
+        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting})
+        self.spider = self.crawler._create_spider('foo')
+        self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
         result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
         return result
 
     @defer.inlineCallbacks
-    def _test_simple_base(self, *mw_classes):
-        result = yield self._get_middleware_result(*mw_classes)
+    def _test_simple_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None):
+        with LogCapture() as log:
+            result = yield self._get_middleware_result(*mw_classes, start_index=start_index)
         self.assertIsInstance(result, collections.abc.Iterable)
         result_list = list(result)
         self.assertEqual(len(result_list), self.RESULT_COUNT)
         self.assertIsInstance(result_list[0], self.ITEM_TYPE)
+        self.assertEqual("downgraded to a non-async" in str(log), downgrade)
 
     @defer.inlineCallbacks
-    def _test_asyncgen_base(self, *mw_classes):
-        result = yield self._get_middleware_result(*mw_classes)
+    def _test_asyncgen_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None):
+        with LogCapture() as log:
+            result = yield self._get_middleware_result(*mw_classes, start_index=start_index)
         self.assertIsInstance(result, collections.abc.AsyncIterator)
         result_list = yield deferred_from_coro(collect_asyncgen(result))
         self.assertEqual(len(result_list), self.RESULT_COUNT)
         self.assertIsInstance(result_list[0], self.ITEM_TYPE)
-
-    @defer.inlineCallbacks
-    def _test_asyncgen_fail(self, *mw_classes):
-        with self.assertRaisesRegex(TypeError, "Synchronous .+ called with an async iterable"):
-            yield self._get_middleware_result(*mw_classes)
+        self.assertEqual("downgraded to a non-async" in str(log), downgrade)
 
 
 class ProcessSpiderOutputSimpleMiddleware:
@@ -152,17 +161,36 @@ class ProcessSpiderOutputSimpleMiddleware:
 
 class ProcessSpiderOutputAsyncGenMiddleware:
     async def process_spider_output(self, response, result, spider):
-        async for r in as_async_generator(result):
+        async for r in result:
             yield r
 
 
 class ProcessSpiderOutputUniversalMiddleware:
     def process_spider_output(self, response, result, spider):
-        @_process_iterable_universal
-        async def process(result):
-            async for r in result:
-                yield r
-        return process(result)
+        for r in result:
+            yield r
+
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result:
+            yield r
+
+
+class ProcessSpiderExceptionSimpleIterableMiddleware:
+    def process_spider_exception(self, response, exception, spider):
+        yield {'foo': 1}
+        yield {'foo': 2}
+        yield {'foo': 3}
+
+
+class ProcessSpiderExceptionAsyncIterableMiddleware:
+    async def process_spider_exception(self, response, exception, spider):
+        yield {'foo': 1}
+        d = defer.Deferred()
+        from twisted.internet import reactor
+        reactor.callLater(0, d.callback, None)
+        await maybe_deferred_to_future(d)
+        yield {'foo': 2}
+        yield {'foo': 3}
 
 
 class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
@@ -183,18 +211,19 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
         return self._test_simple_base(self.MW_SIMPLE)
 
     def test_asyncgen(self):
-        """ Asyncgen mw """
+        """ Asyncgen mw; upgrade """
         return self._test_asyncgen_base(self.MW_ASYNCGEN)
 
     def test_simple_asyncgen(self):
-        """ Simple mw -> asyncgen mw """
+        """ Simple mw -> asyncgen mw; upgrade """
         return self._test_asyncgen_base(self.MW_ASYNCGEN,
                                         self.MW_SIMPLE)
 
     def test_asyncgen_simple(self):
-        """ Asyncgen mw -> simple mw; cannot work """
-        return self._test_asyncgen_fail(self.MW_SIMPLE,
-                                        self.MW_ASYNCGEN)
+        """ Asyncgen mw -> simple mw; upgrade then downgrade """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      self.MW_ASYNCGEN,
+                                      downgrade=True)
 
     def test_universal(self):
         """ Universal mw """
@@ -211,12 +240,12 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
                                       self.MW_SIMPLE)
 
     def test_universal_asyncgen(self):
-        """ Universal mw -> asyncgen mw """
+        """ Universal mw -> asyncgen mw; upgrade """
         return self._test_asyncgen_base(self.MW_ASYNCGEN,
                                         self.MW_UNIVERSAL)
 
     def test_asyncgen_universal(self):
-        """ Asyncgen mw -> universal mw """
+        """ Asyncgen mw -> universal mw; upgrade """
         return self._test_asyncgen_base(self.MW_UNIVERSAL,
                                         self.MW_ASYNCGEN)
 
@@ -229,27 +258,31 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
             yield item
 
     def test_simple(self):
-        """ Simple mw; cannot work """
-        return self._test_asyncgen_fail(self.MW_SIMPLE)
+        """ Simple mw; downgrade """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      downgrade=True)
 
     def test_simple_asyncgen(self):
-        """ Simple mw -> asyncgen mw; cannot work """
-        return self._test_asyncgen_fail(self.MW_ASYNCGEN,
-                                        self.MW_SIMPLE)
+        """ Simple mw -> asyncgen mw; downgrade then upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_SIMPLE,
+                                        downgrade=True)
 
     def test_universal(self):
         """ Universal mw """
         return self._test_asyncgen_base(self.MW_UNIVERSAL)
 
     def test_universal_simple(self):
-        """ Universal mw -> simple mw; cannot work """
-        return self._test_asyncgen_fail(self.MW_SIMPLE,
-                                        self.MW_UNIVERSAL)
+        """ Universal mw -> simple mw; downgrade """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      self.MW_UNIVERSAL,
+                                      downgrade=True)
 
     def test_simple_universal(self):
-        """ Simple mw -> universal mw; cannot work """
-        return self._test_asyncgen_fail(self.MW_UNIVERSAL,
-                                        self.MW_SIMPLE)
+        """ Simple mw -> universal mw; downgrade """
+        return self._test_simple_base(self.MW_UNIVERSAL,
+                                      self.MW_SIMPLE,
+                                      downgrade=True)
 
 
 class ProcessStartRequestsSimpleMiddleware:
@@ -269,13 +302,198 @@ class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
             yield Request(f'https://example.com/{i}', dont_filter=True)
 
     @defer.inlineCallbacks
-    def _get_middleware_result(self, *mw_classes):
-        for mw_cls in mw_classes:
-            self.mwman._add_middleware(mw_cls())
+    def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
+        setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
+        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting})
+        self.spider = self.crawler._create_spider('foo')
+        self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
         start_requests = iter(self._start_requests())
         results = yield self.mwman.process_start_requests(start_requests, self.spider)
         return results
 
     def test_simple(self):
         """ Simple mw """
-        self._test_simple_base(self.MW_SIMPLE)
+        return self._test_simple_base(self.MW_SIMPLE)
+
+
+class UniversalMiddlewareNoSync:
+    async def process_spider_output_async(self, response, result, spider):
+        yield
+
+
+class UniversalMiddlewareBothSync:
+    def process_spider_output(self, response, result, spider):
+        yield
+
+    def process_spider_output_async(self, response, result, spider):
+        yield
+
+
+class UniversalMiddlewareBothAsync:
+    async def process_spider_output(self, response, result, spider):
+        yield
+
+    async def process_spider_output_async(self, response, result, spider):
+        yield
+
+
+class UniversalMiddlewareManagerTest(TestCase):
+    def setUp(self):
+        self.mwman = SpiderMiddlewareManager()
+
+    def test_simple_mw(self):
+        mw = ProcessSpiderOutputSimpleMiddleware
+        self.mwman._add_middleware(mw)
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+
+    def test_async_mw(self):
+        mw = ProcessSpiderOutputAsyncGenMiddleware
+        self.mwman._add_middleware(mw)
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+
+    def test_universal_mw(self):
+        mw = ProcessSpiderOutputUniversalMiddleware
+        self.mwman._add_middleware(mw)
+        self.assertEqual(self.mwman.methods['process_spider_output'][0],
+                         (mw.process_spider_output, mw.process_spider_output_async))
+
+    def test_universal_mw_no_sync(self):
+        with LogCapture() as log:
+            self.mwman._add_middleware(UniversalMiddlewareNoSync)
+        self.assertIn("UniversalMiddlewareNoSync has process_spider_output_async"
+                      " without process_spider_output", str(log))
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], None)
+
+    def test_universal_mw_both_sync(self):
+        mw = UniversalMiddlewareBothSync
+        with LogCapture() as log:
+            self.mwman._add_middleware(mw)
+        self.assertIn("UniversalMiddlewareBothSync.process_spider_output_async "
+                      "is not an async generator function", str(log))
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+
+    def test_universal_mw_both_async(self):
+        with LogCapture() as log:
+            self.mwman._add_middleware(UniversalMiddlewareBothAsync)
+        self.assertIn("UniversalMiddlewareBothAsync.process_spider_output "
+                      "is an async generator function while process_spider_output_async exists",
+                      str(log))
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], None)
+
+
+class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
+    ITEM_TYPE = dict
+    MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
+    MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
+    MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
+
+    def _scrape_func(self, *args, **kwargs):
+        yield {'foo': 1}
+        yield {'foo': 2}
+        yield {'foo': 3}
+
+    @defer.inlineCallbacks
+    def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
+        setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
+        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES': setting})
+        self.spider = self.crawler._create_spider('foo')
+        self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
+        result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
+        return result
+
+    def test_just_builtin(self):
+        return self._test_simple_base()
+
+    def test_builtin_simple(self):
+        return self._test_simple_base(self.MW_SIMPLE, start_index=1000)
+
+    def test_builtin_async(self):
+        """ Upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
+
+    def test_builtin_universal(self):
+        return self._test_simple_base(self.MW_UNIVERSAL, start_index=1000)
+
+    def test_simple_builtin(self):
+        return self._test_simple_base(self.MW_SIMPLE)
+
+    def test_async_builtin(self):
+        """ Upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN)
+
+    def test_universal_builtin(self):
+        return self._test_simple_base(self.MW_UNIVERSAL)
+
+
+class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
+    async def _scrape_func(self, *args, **kwargs):
+        for item in super()._scrape_func():
+            yield item
+
+    def test_just_builtin(self):
+        return self._test_asyncgen_base()
+
+    def test_builtin_simple(self):
+        """ Downgrade """
+        return self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000)
+
+    def test_builtin_async(self):
+        return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
+
+    def test_builtin_universal(self):
+        return self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000)
+
+    def test_simple_builtin(self):
+        """ Downgrade """
+        return self._test_simple_base(self.MW_SIMPLE, downgrade=True)
+
+    def test_async_builtin(self):
+        return self._test_asyncgen_base(self.MW_ASYNCGEN)
+
+    def test_universal_builtin(self):
+        return self._test_asyncgen_base(self.MW_UNIVERSAL)
+
+
+class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase):
+    ITEM_TYPE = dict
+    MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
+    MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
+    MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
+    MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware
+    MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIterableMiddleware
+
+    def _scrape_func(self, *args, **kwargs):
+        1 / 0
+
+    @defer.inlineCallbacks
+    def _test_asyncgen_nodowngrade(self, *mw_classes):
+        with self.assertRaisesRegex(_InvalidOutput, "Async iterable returned from .+ cannot be downgraded"):
+            yield self._get_middleware_result(*mw_classes)
+
+    def test_exc_simple(self):
+        """ Simple exc mw """
+        return self._test_simple_base(self.MW_EXC_SIMPLE)
+
+    def test_exc_async(self):
+        """ Async exc mw """
+        return self._test_asyncgen_base(self.MW_EXC_ASYNCGEN)
+
+    def test_exc_simple_simple(self):
+        """ Simple exc mw -> simple output mw """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      self.MW_EXC_SIMPLE)
+
+    def test_exc_async_async(self):
+        """ Async exc mw -> async output mw """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_EXC_ASYNCGEN)
+
+    def test_exc_simple_async(self):
+        """ Simple exc mw -> async output mw; upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_EXC_SIMPLE)
+
+    def test_exc_async_simple(self):
+        """ Async exc mw -> simple output mw; cannot work as downgrading is not supported """
+        return self._test_asyncgen_nodowngrade(self.MW_SIMPLE,
+                                               self.MW_EXC_ASYNCGEN)
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 088c14ca8..dac246fb6 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -28,6 +28,7 @@ class RecoveryMiddleware:
 class RecoverySpider(Spider):
     name = 'RecoverySpider'
     custom_settings = {
+        'SPIDER_MIDDLEWARES_BASE': {},
         'SPIDER_MIDDLEWARES': {
             RecoveryMiddleware: 10,
         },
@@ -107,6 +108,13 @@ class GeneratorCallbackSpider(Spider):
         raise ImportError()
 
 
+class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider):
+    async def parse(self, response):
+        yield {'test': 1}
+        yield {'test': 2}
+        raise ImportError()
+
+
 # ================================================================================
 # (2.1) exceptions from a spider callback (generator, middleware right after callback)
 class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider):
@@ -360,6 +368,15 @@ class TestSpiderMiddleware(TestCase):
         self.assertIn("Middleware: ImportError exception caught", str(log2))
         self.assertIn("'item_scraped_count': 2", str(log2))
 
+    @defer.inlineCallbacks
+    def test_async_generator_callback(self):
+        """
+        Same as test_generator_callback but with an async callback.
+        """
+        log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider)
+        self.assertIn("Middleware: ImportError exception caught", str(log2))
+        self.assertIn("'item_scraped_count': 2", str(log2))
+
     @defer.inlineCallbacks
     def test_generator_callback_right_after_callback(self):
         """
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index 7abe17c22..9ae66c57c 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -1,7 +1,6 @@
-from twisted.internet.defer import Deferred
 from twisted.trial import unittest
 
-from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.defer import deferred_f_from_coro_f
 
 
@@ -19,52 +18,3 @@ class AsyncgenUtilsTest(unittest.TestCase):
         ag = as_async_generator(range(42))
         results = await collect_asyncgen(ag)
         self.assertEqual(results, list(range(42)))
-
-
-@_process_iterable_universal
-async def process_iterable(iterable):
-    async for i in iterable:
-        yield i * 2
-
-
-@_process_iterable_universal
-async def process_iterable_awaiting(iterable):
-    async for i in iterable:
-        yield i * 2
-        d = Deferred()
-        from twisted.internet import reactor
-        reactor.callLater(0, d.callback, 42)
-        await d
-
-
-class ProcessIterableUniversalTest(unittest.TestCase):
-
-    def test_normal(self):
-        iterable = iter([1, 2, 3])
-        results = list(process_iterable(iterable))
-        self.assertEqual(results, [2, 4, 6])
-
-    @deferred_f_from_coro_f
-    async def test_async(self):
-        iterable = as_async_generator([1, 2, 3])
-        results = await collect_asyncgen(process_iterable(iterable))
-        self.assertEqual(results, [2, 4, 6])
-
-    @deferred_f_from_coro_f
-    async def test_blocking(self):
-        iterable = [1, 2, 3]
-        with self.assertRaisesRegex(RuntimeError, "Synchronously-called function"):
-            list(process_iterable_awaiting(iterable))
-
-    def test_invalid_iterable(self):
-        with self.assertRaisesRegex(TypeError, "Wrong iterable type"):
-            process_iterable(None)
-
-    @deferred_f_from_coro_f
-    async def test_invalid_process(self):
-        @_process_iterable_universal
-        def process_iterable_invalid(iterable):
-            pass
-
-        with self.assertRaisesRegex(ValueError, "process_async returned wrong type"):
-            list(process_iterable_invalid([]))

From e079bffdab11402d1936103ba453e43e97925079 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 11 Jan 2022 19:21:07 +0500
Subject: [PATCH 526/716] Disable logging-fstring-interpolation in pylint.

---
 pylintrc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pylintrc b/pylintrc
index 2cdd6321e..0d29dc709 100644
--- a/pylintrc
+++ b/pylintrc
@@ -49,6 +49,7 @@ disable=abstract-method,
         keyword-arg-before-vararg,
         line-too-long,
         logging-format-interpolation,
+        logging-fstring-interpolation,
         logging-not-lazy,
         lost-exception,
         method-hidden,

From 4bdaa54af4470582845206a9fa21779f80a4b123 Mon Sep 17 00:00:00 2001
From: Georgiy Zatserklianyi <GeorgeA92@users.noreply.github.com>
Date: Fri, 28 Jan 2022 15:39:32 +0200
Subject: [PATCH 527/716] response_httprepr memory issue fixed (#4972)

* response_httprepr replaced by response.body

* unused import deleted

* get_header_size function added

* response size calculation updated

* flake8 codestyle fix

* added counting status code, line breaks to response size

* get_status size: list to tuple, comments added

* test added: comparing new response size counting method with old `len(response_httprepr)`

* downloader stats : unreachable code deleted

* `get_status_size` optimized

* comment added

* tests.test_downloadermiddleware_stats: statement formatting updated

* scrapy.utils.response: `response_httprepr` -> deprecated

* tests.test_downloadermiddleware_stats: flake8 fix
---
 scrapy/downloadermiddlewares/stats.py    | 22 +++++++++++++++++++---
 scrapy/utils/response.py                 |  2 ++
 tests/test_downloadermiddleware_stats.py | 19 +++++++++++++++++++
 3 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py
index 5479cd0e2..25fb1ed9d 100644
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@@ -1,7 +1,22 @@
 from scrapy.exceptions import NotConfigured
+from scrapy.utils.python import global_object_name, to_bytes
 from scrapy.utils.request import request_httprepr
-from scrapy.utils.response import response_httprepr
-from scrapy.utils.python import global_object_name
+
+from twisted.web import http
+
+
+def get_header_size(headers):
+    size = 0
+    for key, value in headers.items():
+        if isinstance(value, (list, tuple)):
+            for v in value:
+                size += len(b": ") + len(key) + len(v)
+    return size + len(b'\r\n') * (len(headers.keys()) - 1)
+
+
+def get_status_size(response_status):
+    return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15
+    # resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
 
 
 class DownloaderStats:
@@ -24,7 +39,8 @@ class DownloaderStats:
     def process_response(self, request, response, spider):
         self.stats.inc_value('downloader/response_count', spider=spider)
         self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
-        reslen = len(response_httprepr(response))
+        reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4
+        # response.body + b"\r\n"+ response.header + b"\r\n" + response.status
         self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
         return response
 
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 8b109dced..741dce350 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -14,6 +14,7 @@ from scrapy.http.response import Response
 
 from twisted.web import http
 from scrapy.utils.python import to_bytes, to_unicode
+from scrapy.utils.decorators import deprecated
 from w3lib import html
 
 
@@ -51,6 +52,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str:
     return f'{status_int} {to_unicode(message)}'
 
 
+@deprecated
 def response_httprepr(response: Response) -> bytes:
     """Return raw HTTP representation (as bytes) of the given response. This
     is provided only for reference, since it's not the exact stream of bytes
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 1f2616e35..9e75f0a50 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -1,8 +1,10 @@
+from itertools import product
 from unittest import TestCase
 
 from scrapy.downloadermiddlewares.stats import DownloaderStats
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
+from scrapy.utils.response import response_httprepr
 from scrapy.utils.test import get_crawler
 
 
@@ -37,6 +39,23 @@ class TestDownloaderStats(TestCase):
         self.mw.process_response(self.req, self.res, self.spider)
         self.assertStatsEqual('downloader/response_count', 1)
 
+    def test_response_len(self):
+        body = (b'', b'not_empty')  # empty/notempty body
+        headers = ({}, {'lang': 'en'}, {'lang': 'en', 'User-Agent': 'scrapy'})  # 0 headers, 1h and 2h
+        test_responses = [  # form test responses with all combinations of body/headers
+            Response(
+                url='scrapytest.org',
+                status=200,
+                body=r[0],
+                headers=r[1]
+            )
+            for r in product(body, headers)
+        ]
+        for test_response in test_responses:
+            self.crawler.stats.set_value('downloader/response_bytes', 0)
+            self.mw.process_response(self.req, test_response, self.spider)
+            self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response)))
+
     def test_process_exception(self):
         self.mw.process_exception(self.req, MyException(), self.spider)
         self.assertStatsEqual('downloader/exception_count', 1)

From 30d5779ea94ed1e9343a4590895a3f5e65e444b9 Mon Sep 17 00:00:00 2001
From: "Sixuan (Cherie) Wu" <73203695+inspurwusixuan@users.noreply.github.com>
Date: Fri, 28 Jan 2022 09:30:30 -0800
Subject: [PATCH 528/716] Fix FEED_URI_PARAMS: custom params throws KeyError
 (#4966)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix FEED_URI_PARAMS: custom params throws KeyError closes #4962

* another try FEED_URI_PARAMS

* add warning message and change default function

* Add tests for FEED_URI_PARAMS

* FEED_URI_PARAMS: warn if the params dict has been modified in-place

* [Doc] FEED_URI_PARAMS: modifying params in-place is deprecated

* Remove whileline

* Rename parameters for lambda function

* Type hints for FeedExporter._get_uri_params

Co-authored-by: Adrián Chaves <adrian@chaves.io>
Co-authored-by: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
---
 docs/topics/feed-exports.rst    |   7 +-
 scrapy/extensions/feedexport.py |  26 +++--
 tests/test_feedexport.py        | 163 ++++++++++++++++++++++++++++++++
 3 files changed, 188 insertions(+), 8 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 116967280..7994027d2 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -322,7 +322,7 @@ Post-Processing
 
 Scrapy provides an option to activate plugins to post-process feeds before they are exported
 to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
-can create your own :ref:`plugins <custom-plugins>`. 
+can create your own :ref:`plugins <custom-plugins>`.
 
 These plugins can be activated through the ``postprocessing`` option of a feed.
 The option must be passed a list of post-processing plugins in the order you want
@@ -366,7 +366,7 @@ Each plugin is a class that must implement the following methods:
 
     Close the target file object.
 
-To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You 
+To pass a parameter to your plugin, use :ref:`feed options <feed-options>`. You
 can then access those parameters from the ``__init__`` method of your plugin.
 
 
@@ -744,6 +744,9 @@ The function signature should be as follows:
    :param spider: source spider of the feed items
    :type spider: scrapy.Spider
 
+   .. caution:: The function should return a new dictionary, modifying
+                the received ``params`` in-place is deprecated.
+
 For example, to include the :attr:`name <scrapy.Spider.name>` of the
 source spider in the feed URI:
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 370723368..e7097b7a1 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -11,14 +11,14 @@ import sys
 import warnings
 from datetime import datetime
 from tempfile import NamedTemporaryFile
-from typing import Any, Optional, Tuple
+from typing import Any, Callable, Optional, Tuple, Union
 from urllib.parse import unquote, urlparse
 
 from twisted.internet import defer, threads
 from w3lib.url import file_uri_to_path
 from zope.interface import implementer, Interface
 
-from scrapy import signals
+from scrapy import signals, Spider
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.extensions.postprocessing import PostProcessingManager
 from scrapy.utils.boto import is_botocore_available
@@ -524,7 +524,12 @@ class FeedExporter:
             raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None")
         return instance
 
-    def _get_uri_params(self, spider, uri_params, slot=None):
+    def _get_uri_params(
+        self,
+        spider: Spider,
+        uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]],
+        slot: Optional[_FeedSlot] = None,
+    ) -> dict:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
@@ -532,9 +537,18 @@ class FeedExporter:
         params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
         params['batch_time'] = utc_now.isoformat().replace(':', '-')
         params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
-        uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
-        uripar_function(params, spider)
-        return params
+        original_params = params.copy()
+        uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params
+        new_params = uripar_function(params, spider)
+        if new_params is None or original_params != params:
+            warnings.warn(
+                'Modifying the params dictionary in-place in the function defined in '
+                'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS '
+                'setting is deprecated. The function must return a new dictionary '
+                'instead.',
+                category=ScrapyDeprecationWarning
+            )
+        return new_params if new_params is not None else params
 
     def _load_filter(self, feed_options):
         # load the item filter if declared else load the default filter class
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 253f3119c..f0acf1941 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2608,3 +2608,166 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
                     ),
                 )
             )
+
+
+class URIParamsTest:
+
+    spider_name = "uri_params_spider"
+
+    def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
+        raise NotImplementedError
+
+    def test_default(self):
+        settings = self.build_settings(
+            uri='file:///tmp/%(name)s',
+        )
+        crawler = get_crawler(settings_dict=settings)
+        feed_exporter = FeedExporter.from_crawler(crawler)
+        spider = scrapy.Spider(self.spider_name)
+        spider.crawler = crawler
+        with warnings.catch_warnings(record=True) as w:
+            feed_exporter.open_spider(spider)
+            messages = tuple(
+                str(item.message) for item in w
+                if item.category is ScrapyDeprecationWarning
+            )
+            self.assertEqual(messages, tuple())
+
+        self.assertEqual(
+            feed_exporter.slots[0].uri,
+            f'file:///tmp/{self.spider_name}'
+        )
+
+    def test_none(self):
+        def uri_params(params, spider):
+            pass
+
+        settings = self.build_settings(
+            uri='file:///tmp/%(name)s',
+            uri_params=uri_params,
+        )
+        crawler = get_crawler(settings_dict=settings)
+        feed_exporter = FeedExporter.from_crawler(crawler)
+        spider = scrapy.Spider(self.spider_name)
+        spider.crawler = crawler
+        with warnings.catch_warnings(record=True) as w:
+            feed_exporter.open_spider(spider)
+            messages = tuple(
+                str(item.message) for item in w
+                if item.category is ScrapyDeprecationWarning
+            )
+            self.assertEqual(
+                messages,
+                (
+                    (
+                        'Modifying the params dictionary in-place in the '
+                        'function defined in the FEED_URI_PARAMS setting or '
+                        'in the uri_params key of the FEEDS setting is '
+                        'deprecated. The function must return a new '
+                        'dictionary instead.'
+                    ),
+                )
+            )
+
+        self.assertEqual(
+            feed_exporter.slots[0].uri,
+            f'file:///tmp/{self.spider_name}'
+        )
+
+    def test_empty_dict(self):
+        def uri_params(params, spider):
+            return {}
+
+        settings = self.build_settings(
+            uri='file:///tmp/%(name)s',
+            uri_params=uri_params,
+        )
+        crawler = get_crawler(settings_dict=settings)
+        feed_exporter = FeedExporter.from_crawler(crawler)
+        spider = scrapy.Spider(self.spider_name)
+        spider.crawler = crawler
+        with warnings.catch_warnings(record=True) as w:
+            with self.assertRaises(KeyError):
+                feed_exporter.open_spider(spider)
+            messages = tuple(
+                str(item.message) for item in w
+                if item.category is ScrapyDeprecationWarning
+            )
+            self.assertEqual(messages, tuple())
+
+    def test_params_as_is(self):
+        def uri_params(params, spider):
+            return params
+
+        settings = self.build_settings(
+            uri='file:///tmp/%(name)s',
+            uri_params=uri_params,
+        )
+        crawler = get_crawler(settings_dict=settings)
+        feed_exporter = FeedExporter.from_crawler(crawler)
+        spider = scrapy.Spider(self.spider_name)
+        spider.crawler = crawler
+        with warnings.catch_warnings(record=True) as w:
+            feed_exporter.open_spider(spider)
+            messages = tuple(
+                str(item.message) for item in w
+                if item.category is ScrapyDeprecationWarning
+            )
+            self.assertEqual(messages, tuple())
+
+        self.assertEqual(
+            feed_exporter.slots[0].uri,
+            f'file:///tmp/{self.spider_name}'
+        )
+
+    def test_custom_param(self):
+        def uri_params(params, spider):
+            return {**params, 'foo': self.spider_name}
+
+        settings = self.build_settings(
+            uri='file:///tmp/%(foo)s',
+            uri_params=uri_params,
+        )
+        crawler = get_crawler(settings_dict=settings)
+        feed_exporter = FeedExporter.from_crawler(crawler)
+        spider = scrapy.Spider(self.spider_name)
+        spider.crawler = crawler
+        with warnings.catch_warnings(record=True) as w:
+            feed_exporter.open_spider(spider)
+            messages = tuple(
+                str(item.message) for item in w
+                if item.category is ScrapyDeprecationWarning
+            )
+            self.assertEqual(messages, tuple())
+
+        self.assertEqual(
+            feed_exporter.slots[0].uri,
+            f'file:///tmp/{self.spider_name}'
+        )
+
+
+class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
+
+    def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
+        extra_settings = {}
+        if uri_params:
+            extra_settings['FEED_URI_PARAMS'] = uri_params
+        return {
+            'FEED_URI': uri,
+            **extra_settings,
+        }
+
+
+class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase):
+
+    def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
+        options = {
+            'format': 'jl',
+        }
+        if uri_params:
+            options['uri_params'] = uri_params
+        return {
+            'FEEDS': {
+                uri: options,
+            },
+        }

From fe43411bc4d0164a0f0ecc596c23b59c99d31f17 Mon Sep 17 00:00:00 2001
From: Laerte <5853172+Laerte@users.noreply.github.com>
Date: Fri, 4 Feb 2022 05:57:57 -0300
Subject: [PATCH 529/716] Fix TypeError on using pathlib.Path as key on FEEDS
 settings (#5384)

---
 scrapy/settings/__init__.py    | 6 +++++-
 tests/test_cmdline/__init__.py | 4 ++++
 tests/test_cmdline/settings.py | 9 +++++++++
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 1fe1e6fd1..6b1ad0828 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -375,9 +375,13 @@ class BaseSettings(MutableMapping):
         return len(self.attributes)
 
     def _to_dict(self):
-        return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
+        return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
                 for k, v in self.items()}
 
+    def _get_key(self, key_value):
+        return (key_value if isinstance(key_value, (bool, float, int, str, type(None)))
+                else str(key_value))
+
     def copy_to_dict(self):
         """
         Make a copy of current settings and convert to a dict.
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 591075a98..8233e0101 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -64,3 +64,7 @@ class CmdlineTest(unittest.TestCase):
         settingsdict = json.loads(settingsstr)
         self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
         self.assertEqual(200, settingsdict[EXT_PATH])
+
+    def test_pathlib_path_as_feeds_key(self):
+        self.assertEqual(self._execute('settings', '--get', 'FEEDS'),
+                         json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}}))
diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py
index 8a719ddf2..b0ac6e98b 100644
--- a/tests/test_cmdline/settings.py
+++ b/tests/test_cmdline/settings.py
@@ -1,5 +1,14 @@
+from pathlib import Path
+
 EXTENSIONS = {
     'tests.test_cmdline.extensions.TestExtension': 0,
 }
 
 TEST1 = 'default'
+
+FEEDS = {
+    Path('items.csv'): {
+        'format': 'csv',
+        'fields': ['price', 'name'],
+    },
+}

From 9be878fc09cf71bb2cb98695f5042cb344bd2e25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 4 Feb 2022 12:27:39 +0100
Subject: [PATCH 530/716] CI: stop using tox-pip-version (#5389)

---
 .github/workflows/checks.yml       | 4 ----
 .github/workflows/tests-ubuntu.yml | 4 ----
 2 files changed, 8 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 80df9469d..98fa44c7f 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -19,7 +19,6 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: pylint
-            TOX_PIP_VERSION: 20.3.3
         - python-version: 3.6
           env:
             TOXENV: typing
@@ -38,8 +37,5 @@ jobs:
     - name: Run check
       env: ${{ matrix.env }}
       run: |
-        if [[ ! -z "$TOX_PIP_VERSION" ]]; then
-            pip install tox-pip-version
-        fi
         pip install -U tox
         tox
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 5ea50e644..1fc8d914b 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -46,7 +46,6 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: extra-deps
-            TOX_PIP_VERSION: 20.3.3
 
     steps:
     - uses: actions/checkout@v2
@@ -73,9 +72,6 @@ jobs:
           $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
           source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
         fi
-        if [[ ! -z "$TOX_PIP_VERSION" ]]; then
-            pip install tox-pip-version
-        fi
         pip install -U tox
         tox
 

From 55ae2109c95e497d4a730afeb9caf71aa78a7723 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 5 Feb 2022 13:02:02 -0300
Subject: [PATCH 531/716] Remove deprecated TextResponse.body_as_unicode

---
 scrapy/http/response/text.py |  9 ---------
 tests/test_http_response.py  | 18 ------------------
 2 files changed, 27 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 27bd55c07..89516b9b6 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -6,7 +6,6 @@ See documentation in docs/topics/request-response.rst
 """
 
 import json
-import warnings
 from contextlib import suppress
 from typing import Generator, Tuple
 from urllib.parse import urljoin
@@ -16,7 +15,6 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
                             http_content_type_encoding, resolve_encoding)
 from w3lib.html import strip_html5_whitespace
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.http.response import Response
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
@@ -66,13 +64,6 @@ class TextResponse(Response):
             or self._body_declared_encoding()
         )
 
-    def body_as_unicode(self):
-        """Return body as unicode"""
-        warnings.warn('Response.body_as_unicode() is deprecated, '
-                      'please use Response.text instead.',
-                      ScrapyDeprecationWarning, stacklevel=2)
-        return self.text
-
     def json(self):
         """
         .. versionadded:: 2.2
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 0ec5257e1..2986f884f 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,10 +1,8 @@
 import unittest
 from unittest import mock
-from warnings import catch_warnings, filterwarnings
 
 from w3lib.encoding import resolve_encoding
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
                          XmlResponse, Headers)
 from scrapy.selector import Selector
@@ -134,9 +132,6 @@ class BaseResponseTest(unittest.TestCase):
         assert isinstance(response.text, str)
         self._assert_response_encoding(response, encoding)
         self.assertEqual(response.body, body_bytes)
-        with catch_warnings():
-            filterwarnings("ignore", category=ScrapyDeprecationWarning)
-            self.assertEqual(response.body_as_unicode(), body_unicode)
         self.assertEqual(response.text, body_unicode)
 
     def _assert_response_encoding(self, response, encoding):
@@ -346,12 +341,6 @@ class TextResponseTest(BaseResponseTest):
         original_string = unicode_string.encode('cp1251')
         r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
 
-        # check body_as_unicode
-        with catch_warnings():
-            filterwarnings("ignore", category=ScrapyDeprecationWarning)
-            self.assertTrue(isinstance(r1.body_as_unicode(), str))
-            self.assertEqual(r1.body_as_unicode(), unicode_string)
-
         # check response.text
         self.assertTrue(isinstance(r1.text, str))
         self.assertEqual(r1.text, unicode_string)
@@ -683,13 +672,6 @@ class TextResponseTest(BaseResponseTest):
         with self.assertRaises(ValueError):
             response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')
 
-    def test_body_as_unicode_deprecation_warning(self):
-        with catch_warnings(record=True) as warnings:
-            r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8')
-            self.assertEqual(r1.body_as_unicode(), 'Hello')
-            self.assertEqual(len(warnings), 1)
-            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
-
     def test_json_response(self):
         json_body = b"""{"ip": "109.187.217.200"}"""
         json_response = self.response_class("http://www.example.com", body=json_body)

From 38d2a154ec79767558f699ec663697ccc7f64ca8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A9ter=20Ferenc=20Gyarmati?= <dev.petergy@gmail.com>
Date: Sun, 6 Feb 2022 18:52:15 +0100
Subject: [PATCH 532/716] docs: use https scheme for each quotes.toscrape.com
 url occurrence

---
 docs/intro/examples.rst         |  2 +-
 docs/intro/overview.rst         |  4 +--
 docs/intro/tutorial.rst         | 52 ++++++++++++++++-----------------
 docs/topics/developer-tools.rst | 20 ++++++-------
 docs/topics/logging.rst         |  2 +-
 docs/topics/settings.rst        |  4 +--
 docs/topics/signals.rst         |  2 +-
 7 files changed, 43 insertions(+), 43 deletions(-)

diff --git a/docs/intro/examples.rst b/docs/intro/examples.rst
index 96363c7d5..edff894c6 100644
--- a/docs/intro/examples.rst
+++ b/docs/intro/examples.rst
@@ -7,7 +7,7 @@ Examples
 The best way to learn is with examples, and Scrapy is no exception. For this
 reason, there is an example Scrapy project named quotesbot_, that you can use to
 play and learn more about Scrapy. It contains two spiders for
-http://quotes.toscrape.com, one using CSS selectors and another one using XPath
+https://quotes.toscrape.com, one using CSS selectors and another one using XPath
 expressions.
 
 The quotesbot_ project is available at: https://github.com/scrapy/quotesbot.
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 405bf845d..f3d652621 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an
 example of a Scrapy Spider using the simplest way to run a spider.
 
 Here's the code for a spider that scrapes famous quotes from website
-http://quotes.toscrape.com, following the pagination::
+https://quotes.toscrape.com, following the pagination::
 
     import scrapy
 
@@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination::
     class QuotesSpider(scrapy.Spider):
         name = 'quotes'
         start_urls = [
-            'http://quotes.toscrape.com/tag/humor/',
+            'https://quotes.toscrape.com/tag/humor/',
         ]
 
         def parse(self, response):
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index ca5856881..5697b9608 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -7,7 +7,7 @@ Scrapy Tutorial
 In this tutorial, we'll assume that Scrapy is already installed on your system.
 If that's not the case, see :ref:`intro-install`.
 
-We are going to scrape `quotes.toscrape.com <http://quotes.toscrape.com/>`_, a website
+We are going to scrape `quotes.toscrape.com <https://quotes.toscrape.com/>`_, a website
 that lists quotes from famous authors.
 
 This tutorial will walk you through these tasks:
@@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named
 
         def start_requests(self):
             urls = [
-                'http://quotes.toscrape.com/page/1/',
-                'http://quotes.toscrape.com/page/2/',
+                'https://quotes.toscrape.com/page/1/',
+                'https://quotes.toscrape.com/page/2/',
             ]
             for url in urls:
                 yield scrapy.Request(url=url, callback=self.parse)
@@ -143,9 +143,9 @@ similar to this::
     2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened
     2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
     2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
-    2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET http://quotes.toscrape.com/robots.txt> (referer: None)
-    2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
-    2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/2/> (referer: None)
+    2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) <GET https://quotes.toscrape.com/robots.txt> (referer: None)
+    2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
+    2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/2/> (referer: None)
     2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html
     2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html
     2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished)
@@ -184,8 +184,8 @@ for your spider::
     class QuotesSpider(scrapy.Spider):
         name = "quotes"
         start_urls = [
-            'http://quotes.toscrape.com/page/1/',
-            'http://quotes.toscrape.com/page/2/',
+            'https://quotes.toscrape.com/page/1/',
+            'https://quotes.toscrape.com/page/2/',
         ]
 
         def parse(self, response):
@@ -207,7 +207,7 @@ Extracting data
 The best way to learn how to extract data with Scrapy is trying selectors
 using the :ref:`Scrapy shell <topics-shell>`. Run::
 
-    scrapy shell 'http://quotes.toscrape.com/page/1/'
+    scrapy shell 'https://quotes.toscrape.com/page/1/'
 
 .. note::
 
@@ -217,18 +217,18 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
 
    On Windows, use double quotes instead::
 
-       scrapy shell "http://quotes.toscrape.com/page/1/"
+       scrapy shell "https://quotes.toscrape.com/page/1/"
 
 You will see something like::
 
     [ ... Scrapy log here ... ]
-    2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None)
+    2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://quotes.toscrape.com/page/1/> (referer: None)
     [s] Available Scrapy objects:
     [s]   scrapy     scrapy module (contains scrapy.Request, scrapy.Selector, etc)
     [s]   crawler    <scrapy.crawler.Crawler object at 0x7fa91d888c90>
     [s]   item       {}
-    [s]   request    <GET http://quotes.toscrape.com/page/1/>
-    [s]   response   <200 http://quotes.toscrape.com/page/1/>
+    [s]   request    <GET https://quotes.toscrape.com/page/1/>
+    [s]   response   <200 https://quotes.toscrape.com/page/1/>
     [s]   settings   <scrapy.settings.Settings object at 0x7fa91d888c10>
     [s]   spider     <DefaultSpider 'default' at 0x7fa91c8af990>
     [s] Useful shortcuts:
@@ -241,7 +241,7 @@ object:
 
 .. invisible-code-block: python
 
-    response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
+    response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html')
 
 >>> response.css('title')
 [<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
@@ -355,7 +355,7 @@ Extracting quotes and authors
 Now that you know a bit about selection and extraction, let's complete our
 spider by writing the code to extract the quotes from the web page.
 
-Each quote in http://quotes.toscrape.com is represented by HTML elements that look
+Each quote in https://quotes.toscrape.com is represented by HTML elements that look
 like this:
 
 .. code-block:: html
@@ -379,7 +379,7 @@ like this:
 Let's open up scrapy shell and play a bit to find out how to extract the data
 we want::
 
-    $ scrapy shell 'http://quotes.toscrape.com'
+    $ scrapy shell 'https://quotes.toscrape.com'
 
 We get a list of selectors for the quote HTML elements with:
 
@@ -444,8 +444,8 @@ in the callback, as you can see below::
     class QuotesSpider(scrapy.Spider):
         name = "quotes"
         start_urls = [
-            'http://quotes.toscrape.com/page/1/',
-            'http://quotes.toscrape.com/page/2/',
+            'https://quotes.toscrape.com/page/1/',
+            'https://quotes.toscrape.com/page/2/',
         ]
 
         def parse(self, response):
@@ -458,9 +458,9 @@ in the callback, as you can see below::
 
 If you run this spider, it will output the extracted data with the log::
 
-    2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
+    2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
     {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
-    2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/>
+    2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
     {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"}
 
 
@@ -505,7 +505,7 @@ Following links
 ===============
 
 Let's say, instead of just scraping the stuff from the first two pages
-from http://quotes.toscrape.com, you want quotes from all the pages in the website.
+from https://quotes.toscrape.com, you want quotes from all the pages in the website.
 
 Now that you know how to extract data from pages, let's see how to follow links
 from them.
@@ -549,7 +549,7 @@ page, extracting data from it::
     class QuotesSpider(scrapy.Spider):
         name = "quotes"
         start_urls = [
-            'http://quotes.toscrape.com/page/1/',
+            'https://quotes.toscrape.com/page/1/',
         ]
 
         def parse(self, response):
@@ -600,7 +600,7 @@ As a shortcut for creating Request objects you can use
     class QuotesSpider(scrapy.Spider):
         name = "quotes"
         start_urls = [
-            'http://quotes.toscrape.com/page/1/',
+            'https://quotes.toscrape.com/page/1/',
         ]
 
         def parse(self, response):
@@ -654,7 +654,7 @@ this time for scraping author information::
     class AuthorSpider(scrapy.Spider):
         name = 'author'
 
-        start_urls = ['http://quotes.toscrape.com/']
+        start_urls = ['https://quotes.toscrape.com/']
 
         def parse(self, response):
             author_page_links = response.css('.author + a')
@@ -727,7 +727,7 @@ with a specific tag, building the URL based on the argument::
         name = "quotes"
 
         def start_requests(self):
-            url = 'http://quotes.toscrape.com/'
+            url = 'https://quotes.toscrape.com/'
             tag = getattr(self, 'tag', None)
             if tag is not None:
                 url = url + 'tag/' + tag
@@ -747,7 +747,7 @@ with a specific tag, building the URL based on the argument::
 
 If you pass the ``tag=humor`` argument to this spider, you'll notice that it
 will only visit URLs from the ``humor`` tag, such as
-``http://quotes.toscrape.com/tag/humor``.
+``https://quotes.toscrape.com/tag/humor``.
 
 You can :ref:`learn more about handling spider arguments here <spiderargs>`.
 
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index 057b1ec62..96475899f 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class=
 "text"`` we will see the quote-text we clicked on. The `Inspector` lets you
 copy XPaths to selected elements. Let's try it out.
 
-First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
+First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal:
 
 .. code-block:: none
 
-    $ scrapy shell "http://quotes.toscrape.com/"
+    $ scrapy shell "https://quotes.toscrape.com/"
 
 Then, back to your web browser, right-click on the ``span`` tag, select
 ``Copy > XPath`` and paste it in the Scrapy shell like so:
 
 .. invisible-code-block: python
 
-    response = load_response('http://quotes.toscrape.com/', 'quotes.html')
+    response = load_response('https://quotes.toscrape.com/', 'quotes.html')
 
 >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
 ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
@@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the
 type ``json``.
 
 If we click on this request, we see that the request URL is
-``http://quotes.toscrape.com/api/quotes?page=1`` and the response
+``https://quotes.toscrape.com/api/quotes?page=1`` and the response
 is a JSON-object that contains our quotes. We can also right-click
 on the request and open ``Open in new tab`` to get a better overview.
 
@@ -247,7 +247,7 @@ also request each page to get every quote on the site::
         name = 'quote'
         allowed_domains = ['quotes.toscrape.com']
         page = 1
-        start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
+        start_urls = ['https://quotes.toscrape.com/api/quotes?page=1']
 
         def parse(self, response):
             data = json.loads(response.text)
@@ -255,7 +255,7 @@ also request each page to get every quote on the site::
                 yield {"quote": quote["text"]}
             if data["has_next"]:
                 self.page += 1
-                url = f"http://quotes.toscrape.com/api/quotes?page={self.page}"
+                url = f"https://quotes.toscrape.com/api/quotes?page={self.page}"
                 yield scrapy.Request(url=url, callback=self.parse)
 
 This spider starts at the first page of the quotes-API. With each
@@ -280,7 +280,7 @@ request::
     from scrapy import Request
 
     request = Request.from_curl(
-        "curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
+        "curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
         "la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
         "pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
         "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
@@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down
 to identifying the correct request and replicating it in your spider.
 
 .. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
-.. _quotes.toscrape.com: http://quotes.toscrape.com
-.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
-.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
+.. _quotes.toscrape.com: https://quotes.toscrape.com
+.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll
+.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10
 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
 
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index d593c74c6..3bf23d5f5 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -218,7 +218,7 @@ For example, let's say you're scraping a website which returns many
 HTTP 404 and 500 responses, and you want to hide all messages like this::
 
     2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring
-    response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code
+    response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code
     is not handled or not allowed
 
 The first thing to note is a logger name - it is in brackets:
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index f6c95c502..4e105642d 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1597,7 +1597,7 @@ In order to use the reactor installed by Scrapy::
         def start_requests(self):
             reactor.callLater(self.timeout, self.stop)
 
-            urls = ['http://quotes.toscrape.com/page/1']
+            urls = ['https://quotes.toscrape.com/page/1']
             for url in urls:
                 yield scrapy.Request(url=url, callback=self.parse)
 
@@ -1625,7 +1625,7 @@ which raises :exc:`Exception`, becomes::
             from twisted.internet import reactor
             reactor.callLater(self.timeout, self.stop)
 
-            urls = ['http://quotes.toscrape.com/page/1']
+            urls = ['https://quotes.toscrape.com/page/1']
             for url in urls:
                 yield scrapy.Request(url=url, callback=self.parse)
 
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 63ad3a9ad..2fbd0b51c 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -60,7 +60,7 @@ Let's take an example::
 
     class SignalSpider(scrapy.Spider):
         name = 'signals'
-        start_urls = ['http://quotes.toscrape.com/page/1/']
+        start_urls = ['https://quotes.toscrape.com/page/1/']
 
         @classmethod
         def from_crawler(cls, crawler, *args, **kwargs):

From bbfa185664cef79299b48cb0ae22065439bb07fc Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 6 Feb 2022 18:12:28 -0300
Subject: [PATCH 533/716] Remove deprecated BaseItem class

---
 scrapy/exporters.py  |  4 +--
 scrapy/item.py       | 34 ++------------------
 scrapy/utils/misc.py |  4 +--
 tests/test_item.py   | 75 +-------------------------------------------
 4 files changed, 8 insertions(+), 109 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 36cca2d05..1c26e81db 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -13,7 +13,7 @@ from xml.sax.saxutils import XMLGenerator
 from itemadapter import is_item, ItemAdapter
 
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.item import _BaseItem
+from scrapy.item import Item
 from scrapy.utils.python import is_listlike, to_bytes, to_unicode
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
@@ -315,7 +315,7 @@ class PythonItemExporter(BaseItemExporter):
         return serializer(value)
 
     def _serialize_value(self, value):
-        if isinstance(value, _BaseItem):
+        if isinstance(value, Item):
             return self.export_item(value)
         elif is_item(value):
             return dict(self._serialize_item(value))
diff --git a/scrapy/item.py b/scrapy/item.py
index 2ccd7ad18..839bee3fa 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -15,39 +15,11 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
 
-class _BaseItem(object_ref):
-    """
-    Temporary class used internally to avoid the deprecation
-    warning raised by isinstance checks using BaseItem.
-    """
-    pass
-
-
-class _BaseItemMeta(ABCMeta):
-    def __instancecheck__(cls, instance):
-        if cls is BaseItem:
-            warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
-                 ScrapyDeprecationWarning, stacklevel=2)
-        return super().__instancecheck__(instance)
-
-
-class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
-    """
-    Deprecated, please use :class:`scrapy.item.Item` instead
-    """
-
-    def __new__(cls, *args, **kwargs):
-        if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
-            warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
-                 ScrapyDeprecationWarning, stacklevel=2)
-        return super().__new__(cls, *args, **kwargs)
-
-
 class Field(dict):
     """Container of field metadata"""
 
 
-class ItemMeta(_BaseItemMeta):
+class ItemMeta(ABCMeta):
     """Metaclass_ of :class:`Item` that handles field definitions.
 
     .. _metaclass: https://realpython.com/python-metaclasses
@@ -74,7 +46,7 @@ class ItemMeta(_BaseItemMeta):
         return super().__new__(mcs, class_name, bases, new_attrs)
 
 
-class DictItem(MutableMapping, BaseItem):
+class DictItem(MutableMapping, object_ref):
 
     fields: Dict[str, Field] = {}
 
@@ -118,7 +90,7 @@ class DictItem(MutableMapping, BaseItem):
     def __iter__(self):
         return iter(self._values)
 
-    __hash__ = BaseItem.__hash__
+    __hash__ = object_ref.__hash__
 
     def keys(self):
         return self._values.keys()
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 11c4206c2..1221b39b2 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -14,11 +14,11 @@ from w3lib.html import replace_entities
 
 from scrapy.utils.datatypes import LocalWeakReferencedCache
 from scrapy.utils.python import flatten, to_unicode
-from scrapy.item import _BaseItem
+from scrapy.item import Item
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
 
 
-_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
+_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
 
 
 def arg_to_iter(arg):
diff --git a/tests/test_item.py b/tests/test_item.py
index c94bb44af..7d82fbffe 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -3,7 +3,7 @@ from unittest import mock
 from warnings import catch_warnings, filterwarnings
 
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta
+from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
 
 
 class ItemTest(unittest.TestCase):
@@ -318,79 +318,6 @@ class DictItemTest(unittest.TestCase):
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
 
-class BaseItemTest(unittest.TestCase):
-
-    def test_isinstance_check(self):
-
-        class SubclassedBaseItem(BaseItem):
-            pass
-
-        class SubclassedItem(Item):
-            pass
-
-        with catch_warnings():
-            filterwarnings("ignore", category=ScrapyDeprecationWarning)
-            self.assertTrue(isinstance(BaseItem(), BaseItem))
-            self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))
-            self.assertTrue(isinstance(Item(), BaseItem))
-            self.assertTrue(isinstance(SubclassedItem(), BaseItem))
-
-            # make sure internal checks using private _BaseItem class succeed
-            self.assertTrue(isinstance(BaseItem(), _BaseItem))
-            self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))
-            self.assertTrue(isinstance(Item(), _BaseItem))
-            self.assertTrue(isinstance(SubclassedItem(), _BaseItem))
-
-    def test_deprecation_warning(self):
-        """
-        Make sure deprecation warnings are logged whenever BaseItem is used,
-        either instantiated or in an isinstance check
-        """
-        with catch_warnings(record=True) as warnings:
-            BaseItem()
-            self.assertEqual(len(warnings), 1)
-            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
-
-        with catch_warnings(record=True) as warnings:
-
-            class SubclassedBaseItem(BaseItem):
-                pass
-
-            SubclassedBaseItem()
-            self.assertEqual(len(warnings), 1)
-            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
-
-        with catch_warnings(record=True) as warnings:
-            self.assertFalse(isinstance("foo", BaseItem))
-            self.assertEqual(len(warnings), 1)
-            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
-
-        with catch_warnings(record=True) as warnings:
-            self.assertTrue(isinstance(BaseItem(), BaseItem))
-            self.assertEqual(len(warnings), 1)
-            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
-
-
-class ItemNoDeprecationWarningTest(unittest.TestCase):
-    def test_no_deprecation_warning(self):
-        """
-        Make sure deprecation warnings are NOT logged whenever BaseItem subclasses are used.
-        """
-        class SubclassedItem(Item):
-            pass
-
-        with catch_warnings(record=True) as warnings:
-            Item()
-            SubclassedItem()
-            _BaseItem()
-            self.assertFalse(isinstance("foo", _BaseItem))
-            self.assertFalse(isinstance("foo", Item))
-            self.assertFalse(isinstance("foo", SubclassedItem))
-            self.assertTrue(isinstance(_BaseItem(), _BaseItem))
-            self.assertTrue(isinstance(Item(), Item))
-            self.assertTrue(isinstance(SubclassedItem(), SubclassedItem))
-            self.assertEqual(len(warnings), 0)
-
 
 if __name__ == "__main__":
     unittest.main()

From c8c1edd43b04ce7a2d9b1da198af26ba271cb1d6 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 6 Feb 2022 18:27:41 -0300
Subject: [PATCH 534/716] Flake8 adjustments

---
 tests/test_item.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_item.py b/tests/test_item.py
index 7d82fbffe..a12e425e0 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,6 +1,6 @@
 import unittest
 from unittest import mock
-from warnings import catch_warnings, filterwarnings
+from warnings import catch_warnings
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
@@ -318,6 +318,5 @@ class DictItemTest(unittest.TestCase):
             self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
 
 
-
 if __name__ == "__main__":
     unittest.main()

From fca49cca929de035fb5d179d83f7f79da22fd205 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 6 Feb 2022 18:31:55 -0300
Subject: [PATCH 535/716] Remove deprecated DictItem class

---
 docs/conf.py       |  1 -
 scrapy/item.py     | 55 +++++++++++++++++++---------------------------
 tests/test_item.py | 31 +-------------------------
 3 files changed, 23 insertions(+), 64 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 406c4d94a..d5e139e66 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -272,7 +272,6 @@ coverage_ignore_pyobjects = [
     r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]',  # helper functions
 
     # Never documented before, and deprecated now.
-    r'^scrapy\.item\.DictItem$',
     r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
 
     # Implementation detail of LxmlLinkExtractor
diff --git a/scrapy/item.py b/scrapy/item.py
index 839bee3fa..2521ac829 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -9,9 +9,7 @@ from collections.abc import MutableMapping
 from copy import deepcopy
 from pprint import pformat
 from typing import Dict
-from warnings import warn
 
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
 
@@ -46,15 +44,30 @@ class ItemMeta(ABCMeta):
         return super().__new__(mcs, class_name, bases, new_attrs)
 
 
-class DictItem(MutableMapping, object_ref):
+class Item(MutableMapping, object_ref, metaclass=ItemMeta):
+    """
+    Base class for scraped items.
 
-    fields: Dict[str, Field] = {}
+    In Scrapy, an object is considered an ``item`` if it is an instance of either
+    :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
+    spider callback is evaluated, only instances of :class:`Item` or
+    :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
 
-    def __new__(cls, *args, **kwargs):
-        if issubclass(cls, DictItem) and not issubclass(cls, Item):
-            warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
-                 ScrapyDeprecationWarning, stacklevel=2)
-        return super().__new__(cls, *args, **kwargs)
+    If you need instances of a custom class to be considered items by Scrapy,
+    you must inherit from either :class:`Item` or :class:`dict`.
+
+    Items must declare :class:`Field` attributes, which are processed and stored
+    in the ``fields`` attribute. This restricts the set of allowed field names
+    and prevents typos, raising ``KeyError`` when referring to undefined fields.
+    Additionally, fields can be used to define metadata and control the way
+    data is processed internally. Please refer to the :ref:`documentation
+    about fields <topics-items-fields>` for additional information.
+
+    Unlike instances of :class:`dict`, instances of :class:`Item` may be
+    :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
+    """
+
+    fields: Dict[str, Field]
 
     def __init__(self, *args, **kwargs):
         self._values = {}
@@ -105,27 +118,3 @@ class DictItem(MutableMapping, object_ref):
         """Return a :func:`~copy.deepcopy` of this item.
         """
         return deepcopy(self)
-
-
-class Item(DictItem, metaclass=ItemMeta):
-    """
-    Base class for scraped items.
-
-    In Scrapy, an object is considered an ``item`` if it is an instance of either
-    :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a
-    spider callback is evaluated, only instances of :class:`Item` or
-    :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
-
-    If you need instances of a custom class to be considered items by Scrapy,
-    you must inherit from either :class:`Item` or :class:`dict`.
-
-    Items must declare :class:`Field` attributes, which are processed and stored
-    in the ``fields`` attribute. This restricts the set of allowed field names
-    and prevents typos, raising ``KeyError`` when referring to undefined fields.
-    Additionally, fields can be used to define metadata and control the way
-    data is processed internally. Please refer to the :ref:`documentation
-    about fields <topics-items-fields>` for additional information.
-
-    Unlike instances of :class:`dict`, instances of :class:`Item` may be
-    :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
-    """
diff --git a/tests/test_item.py b/tests/test_item.py
index a12e425e0..25f2aea0a 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,9 +1,7 @@
 import unittest
 from unittest import mock
-from warnings import catch_warnings
 
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
+from scrapy.item import ABCMeta, Field, Item, ItemMeta
 
 
 class ItemTest(unittest.TestCase):
@@ -254,18 +252,6 @@ class ItemTest(unittest.TestCase):
         item['tags'].append('tag2')
         assert item['tags'] != copied_item['tags']
 
-    def test_dictitem_deprecation_warning(self):
-        """Make sure the DictItem deprecation warning is not issued for
-        Item"""
-        with catch_warnings(record=True) as warnings:
-            Item()
-            self.assertEqual(len(warnings), 0)
-
-            class SubclassedItem(Item):
-                pass
-            SubclassedItem()
-            self.assertEqual(len(warnings), 0)
-
 
 class ItemMetaTest(unittest.TestCase):
 
@@ -303,20 +289,5 @@ class ItemMetaClassCellRegression(unittest.TestCase):
                 super().__init__(*args, **kwargs)
 
 
-class DictItemTest(unittest.TestCase):
-
-    def test_deprecation_warning(self):
-        with catch_warnings(record=True) as warnings:
-            DictItem()
-            self.assertEqual(len(warnings), 1)
-            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
-        with catch_warnings(record=True) as warnings:
-            class SubclassedDictItem(DictItem):
-                pass
-            SubclassedDictItem()
-            self.assertEqual(len(warnings), 1)
-            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
-
-
 if __name__ == "__main__":
     unittest.main()

From b282a7af012a4804eb91bdd850df3b86065b3fd6 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Tue, 8 Feb 2022 01:25:08 +0500
Subject: [PATCH 536/716] Temporarily pin Twisted to an older version in CI
 (#5401)

---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index 2031a2d92..cf7855cf9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -19,6 +19,8 @@ deps =
     mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # Extras
     botocore>=1.4.87
+    # Temporary until the tests are updated
+    Twisted<22.1.0
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID

From 4bda0976b28917405f54c781afda5ea55b65b16b Mon Sep 17 00:00:00 2001
From: Laerte <5853172+Laerte@users.noreply.github.com>
Date: Tue, 8 Feb 2022 10:57:19 -0300
Subject: [PATCH 537/716] Fix csviter call, add parse_rows test (#5394)

---
 scrapy/spiders/feed.py |  2 +-
 tests/test_spider.py   | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index bef2d6b24..79e12e030 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -123,7 +123,7 @@ class CSVFeedSpider(Spider):
         process_results methods for pre and post-processing purposes.
         """
 
-        for row in csviter(response, self.delimiter, self.headers, self.quotechar):
+        for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar):
             ret = iterate_spider_output(self.parse_row(response, row))
             for result_item in self.process_results(response, ret):
                 yield result_item
diff --git a/tests/test_spider.py b/tests/test_spider.py
index a7c3ee048..689349999 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -21,6 +21,7 @@ from scrapy.spiders import (
 )
 from scrapy.linkextractors import LinkExtractor
 from scrapy.utils.test import get_crawler
+from tests import get_testdata
 
 
 class SpiderTest(unittest.TestCase):
@@ -167,6 +168,23 @@ class CSVFeedSpiderTest(SpiderTest):
 
     spider_class = CSVFeedSpider
 
+    def test_parse_rows(self):
+        body = get_testdata('feeds', 'feed-sample6.csv')
+        response = Response("http://example.org/dummy.csv", body=body)
+
+        class _CrawlSpider(self.spider_class):
+            name = "test"
+            delimiter = ","
+            quotechar = "'"
+
+            def parse_row(self, response, row):
+                return row
+
+        spider = _CrawlSpider()
+        rows = list(spider.parse_rows(response))
+        assert rows[0] == {'id': '1', 'name': 'alpha', 'value': 'foobar'}
+        assert len(rows) == 4
+
 
 class CrawlSpiderTest(SpiderTest):
 

From fd55f62207bbbb18d7758c8e2ef46fe9115eb2c5 Mon Sep 17 00:00:00 2001
From: Raihan Nismara <31585789+raihan71@users.noreply.github.com>
Date: Tue, 8 Feb 2022 21:36:25 +0700
Subject: [PATCH 538/716] Update Logo in README.rst (#5258)

---
 README.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 05f10bb6c..6b563d638 100644
--- a/README.rst
+++ b/README.rst
@@ -1,5 +1,4 @@
-.. image:: /artwork/scrapy-logo.jpg
-   :width: 400px
+.. image:: https://scrapy.org/img/scrapylogo.png
    
 ======
 Scrapy

From 1e1cfc26dbdde5fc3035169884c4d1218844d0de Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 8 Feb 2022 21:01:16 +0500
Subject: [PATCH 539/716] Copy resource classes from
 twisted.web.test.test_webclient.

---
 pytest.ini                        |  2 -
 tests/mockserver.py               | 74 ++++++++++++++++++++++++++++---
 tests/test_downloader_handlers.py | 12 +++--
 tests/test_webclient.py           | 18 ++++----
 4 files changed, 86 insertions(+), 20 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index fa5d6b34f..ae2ed2029 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,5 +21,3 @@ addopts =
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
     only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
-filterwarnings=
-    ignore::DeprecationWarning:twisted.web.test.test_webclient
diff --git a/tests/mockserver.py b/tests/mockserver.py
index ab9aec6a6..72d7e0241 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -14,10 +14,9 @@ from twisted.internet import defer, reactor, ssl
 from twisted.internet.task import deferLater
 from twisted.names import dns, error
 from twisted.names.server import DNSServerFactory
-from twisted.web.resource import EncodingResourceWrapper, Resource
+from twisted.web import resource, server
 from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site
 from twisted.web.static import File
-from twisted.web.test.test_webclient import PayloadResource
 from twisted.web.util import redirectTo
 
 from scrapy.utils.python import to_bytes, to_unicode
@@ -35,7 +34,70 @@ def getarg(request, name, default=None, type=None):
         return default
 
 
-class LeafResource(Resource):
+# most of the following resources are copied from twisted.web.test.test_webclient
+class ForeverTakingResource(resource.Resource):
+    """
+    L{ForeverTakingResource} is a resource which never finishes responding
+    to requests.
+    """
+
+    def __init__(self, write=False):
+        resource.Resource.__init__(self)
+        self._write = write
+
+    def render(self, request):
+        if self._write:
+            request.write(b"some bytes")
+        return server.NOT_DONE_YET
+
+
+class ErrorResource(resource.Resource):
+    def render(self, request):
+        request.setResponseCode(401)
+        if request.args.get(b"showlength"):
+            request.setHeader(b"content-length", b"0")
+        return b""
+
+
+class NoLengthResource(resource.Resource):
+    def render(self, request):
+        return b"nolength"
+
+
+class HostHeaderResource(resource.Resource):
+    """
+    A testing resource which renders itself as the value of the host header
+    from the request.
+    """
+
+    def render(self, request):
+        return request.requestHeaders.getRawHeaders(b"host")[0]
+
+
+class PayloadResource(resource.Resource):
+    """
+    A testing resource which renders itself as the contents of the request body
+    as long as the request body is 100 bytes long, otherwise which renders
+    itself as C{"ERROR"}.
+    """
+
+    def render(self, request):
+        data = request.content.read()
+        contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0]
+        if len(data) != 100 or int(contentLength) != 100:
+            return b"ERROR"
+        return data
+
+
+class BrokenDownloadResource(resource.Resource):
+    def render(self, request):
+        # only sends 3 bytes even though it claims to send 5
+        request.setHeader(b"content-length", b"5")
+        request.write(b"abc")
+        return b""
+
+
+class LeafResource(resource.Resource):
 
     isLeaf = True
 
@@ -175,10 +237,10 @@ class ArbitraryLengthPayloadResource(LeafResource):
         return request.content.read()
 
 
-class Root(Resource):
+class Root(resource.Resource):
 
     def __init__(self):
-        Resource.__init__(self)
+        resource.Resource.__init__(self)
         self.putChild(b"status", Status())
         self.putChild(b"follow", Follow())
         self.putChild(b"delay", Delay())
@@ -187,7 +249,7 @@ class Root(Resource):
         self.putChild(b"raw", Raw())
         self.putChild(b"echo", Echo())
         self.putChild(b"payload", PayloadResource())
-        self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
+        self.putChild(b"xpayload", resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
         self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
         try:
             from tests import tests_datadir
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index a1ea4c679..2bb53950d 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -15,8 +15,6 @@ from twisted.trial import unittest
 from twisted.web import resource, server, static, util
 from twisted.web._newclient import ResponseFailed
 from twisted.web.http import _DataLoss
-from twisted.web.test.test_webclient import (ForeverTakingResource, HostHeaderResource,
-                                             NoLengthResource, PayloadResource)
 from w3lib.url import path_to_file_uri
 
 from scrapy.core.downloader.handlers import DownloadHandlers
@@ -34,7 +32,15 @@ from scrapy.spiders import Spider
 from scrapy.utils.misc import create_instance
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler, skip_if_no_boto
-from tests.mockserver import MockServer, ssl_context_factory, Echo
+from tests.mockserver import (
+    Echo,
+    ForeverTakingResource,
+    HostHeaderResource,
+    MockServer,
+    NoLengthResource,
+    PayloadResource,
+    ssl_context_factory,
+)
 from tests.spiders import SingleRequestSpider
 
 
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 6e4cb9b6e..a6d55cb38 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -21,14 +21,6 @@ except ImportError:
 from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
 from twisted.internet.defer import inlineCallbacks
-from twisted.web.test.test_webclient import (
-    ForeverTakingResource,
-    ErrorResource,
-    NoLengthResource,
-    HostHeaderResource,
-    PayloadResource,
-    BrokenDownloadResource,
-)
 
 from scrapy.core.downloader import webclient as client
 from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
@@ -36,7 +28,15 @@ from scrapy.http import Request, Headers
 from scrapy.settings import Settings
 from scrapy.utils.misc import create_instance
 from scrapy.utils.python import to_bytes, to_unicode
-from tests.mockserver import ssl_context_factory
+from tests.mockserver import (
+    BrokenDownloadResource,
+    ErrorResource,
+    ForeverTakingResource,
+    HostHeaderResource,
+    NoLengthResource,
+    PayloadResource,
+    ssl_context_factory,
+)
 
 
 def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):

From 77547a1ab554a5b9afa7d7a343d8f90ef1d4cfe8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 8 Feb 2022 21:06:02 +0500
Subject: [PATCH 540/716] Revert "Temporarily pin Twisted to an older version
 in CI (#5401)"

This reverts commit b282a7af012a4804eb91bdd850df3b86065b3fd6.
---
 tox.ini | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index cf7855cf9..2031a2d92 100644
--- a/tox.ini
+++ b/tox.ini
@@ -19,8 +19,6 @@ deps =
     mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # Extras
     botocore>=1.4.87
-    # Temporary until the tests are updated
-    Twisted<22.1.0
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID

From e2e2ffd0d162cfed5a2e82e9fb9472dbf233c919 Mon Sep 17 00:00:00 2001
From: Alex <alexpdev@protonmail.com>
Date: Wed, 9 Feb 2022 11:52:07 -0800
Subject: [PATCH 541/716] Move from optparse to argparse (#5374)

---
 scrapy/cmdline.py            | 14 +++----
 scrapy/commands/__init__.py  | 75 ++++++++++++++++++++++++------------
 scrapy/commands/check.py     |  8 ++--
 scrapy/commands/fetch.py     | 10 ++---
 scrapy/commands/genspider.py | 20 +++++-----
 scrapy/commands/parse.py     | 44 ++++++++++-----------
 scrapy/commands/settings.py  | 20 +++++-----
 scrapy/commands/shell.py     | 12 +++---
 scrapy/commands/version.py   |  4 +-
 scrapy/commands/view.py      |  3 +-
 tests/test_command_parse.py  | 19 +++++++++
 tests/test_commands.py       | 38 ++++++++++++++----
 12 files changed, 168 insertions(+), 99 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 91482ce01..491c4beab 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -1,13 +1,13 @@
 import sys
 import os
-import optparse
+import argparse
 import cProfile
 import inspect
 import pkg_resources
 
 import scrapy
 from scrapy.crawler import CrawlerProcess
-from scrapy.commands import ScrapyCommand
+from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
 from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
@@ -123,8 +123,6 @@ def execute(argv=None, settings=None):
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
     cmdname = _pop_command_name(argv)
-    parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
-                                   conflict_handler='resolve')
     if not cmdname:
         _print_commands(settings, inproject)
         sys.exit(0)
@@ -133,12 +131,14 @@ def execute(argv=None, settings=None):
         sys.exit(2)
 
     cmd = cmds[cmdname]
-    parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
-    parser.description = cmd.long_desc()
+    parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
+                                     usage=f"scrapy {cmdname} {cmd.syntax()}",
+                                     conflict_handler='resolve',
+                                     description=cmd.long_desc())
     settings.setdict(cmd.default_settings, priority='command')
     cmd.settings = settings
     cmd.add_options(parser)
-    opts, args = parser.parse_args(args=argv[1:])
+    opts, args = parser.parse_known_args(args=argv[1:])
     _run_print_help(parser, cmd.process_options, args, opts)
 
     cmd.crawler_process = CrawlerProcess(settings)
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 5f1dabd33..fb304b8c0 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -2,7 +2,7 @@
 Base class for Scrapy commands
 """
 import os
-from optparse import OptionGroup
+import argparse
 from typing import Any, Dict
 
 from twisted.python import failure
@@ -59,22 +59,20 @@ class ScrapyCommand:
         """
         Populate option parse with options available for this command
         """
-        group = OptionGroup(parser, "Global Options")
-        group.add_option("--logfile", metavar="FILE",
-                         help="log file. if omitted stderr will be used")
-        group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
-                         help=f"log level (default: {self.settings['LOG_LEVEL']})")
-        group.add_option("--nolog", action="store_true",
-                         help="disable logging completely")
-        group.add_option("--profile", metavar="FILE", default=None,
-                         help="write python cProfile stats to FILE")
-        group.add_option("--pidfile", metavar="FILE",
-                         help="write process ID to FILE")
-        group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
-                         help="set/override setting (may be repeated)")
-        group.add_option("--pdb", action="store_true", help="enable pdb on failure")
-
-        parser.add_option_group(group)
+        group = parser.add_argument_group(title='Global Options')
+        group.add_argument("--logfile", metavar="FILE",
+                           help="log file. if omitted stderr will be used")
+        group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
+                           help=f"log level (default: {self.settings['LOG_LEVEL']})")
+        group.add_argument("--nolog", action="store_true",
+                           help="disable logging completely")
+        group.add_argument("--profile", metavar="FILE", default=None,
+                           help="write python cProfile stats to FILE")
+        group.add_argument("--pidfile", metavar="FILE",
+                           help="write process ID to FILE")
+        group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
+                           help="set/override setting (may be repeated)")
+        group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
 
     def process_options(self, args, opts):
         try:
@@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand):
     """
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
-                          help="set spider argument (may be repeated)")
-        parser.add_option("-o", "--output", metavar="FILE", action="append",
-                          help="append scraped items to the end of FILE (use - for stdout)")
-        parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
-                          help="dump scraped items into FILE, overwriting any existing file")
-        parser.add_option("-t", "--output-format", metavar="FORMAT",
-                          help="format to use for dumping items")
+        parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
+                            help="set spider argument (may be repeated)")
+        parser.add_argument("-o", "--output", metavar="FILE", action="append",
+                            help="append scraped items to the end of FILE (use - for stdout)")
+        parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
+                            help="dump scraped items into FILE, overwriting any existing file")
+        parser.add_argument("-t", "--output-format", metavar="FORMAT",
+                            help="format to use for dumping items")
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
@@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand):
                 opts.overwrite_output,
             )
             self.settings.set('FEEDS', feeds, priority='cmdline')
+
+
+class ScrapyHelpFormatter(argparse.HelpFormatter):
+    """
+    Help Formatter for scrapy command line help messages.
+    """
+    def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
+        super().__init__(prog, indent_increment=indent_increment,
+                         max_help_position=max_help_position, width=width)
+
+    def _join_parts(self, part_strings):
+        parts = self.format_part_strings(part_strings)
+        return super()._join_parts(parts)
+
+    def format_part_strings(self, part_strings):
+        """
+        Underline and title case command line help message headers.
+        """
+        if part_strings and part_strings[0].startswith("usage: "):
+            part_strings[0] = "Usage\n=====\n  " + part_strings[0][len('usage: '):]
+        headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
+        for index in headings[::-1]:
+            char = '-' if "Global Options" in part_strings[index] else '='
+            part_strings[index] = part_strings[index][:-2].title()
+            underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
+            part_strings.insert(index + 1, underline)
+        return part_strings
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index ae21d86e6..a16f4beb7 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -49,10 +49,10 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("-l", "--list", dest="list", action="store_true",
-                          help="only list contracts, without checking them")
-        parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
-                          help="print contract tests for all spiders")
+        parser.add_argument("-l", "--list", dest="list", action="store_true",
+                            help="only list contracts, without checking them")
+        parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
+                            help="print contract tests for all spiders")
 
     def run(self, args, opts):
         # load contracts
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 95f87e8c3..9b2ebb37f 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -26,11 +26,11 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("--spider", dest="spider", help="use this spider")
-        parser.add_option("--headers", dest="headers", action="store_true",
-                          help="print response HTTP headers instead of body")
-        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
-                          help="do not handle HTTP 3xx status codes and print response as-is")
+        parser.add_argument("--spider", dest="spider", help="use this spider")
+        parser.add_argument("--headers", dest="headers", action="store_true",
+                            help="print response HTTP headers instead of body")
+        parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
+                            help="do not handle HTTP 3xx status codes and print response as-is")
 
     def _print_headers(self, headers, prefix):
         for key, values in headers.items():
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 2082a4974..ed5f588e9 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -44,16 +44,16 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("-l", "--list", dest="list", action="store_true",
-                          help="List available templates")
-        parser.add_option("-e", "--edit", dest="edit", action="store_true",
-                          help="Edit spider after creating it")
-        parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
-                          help="Dump template to standard output")
-        parser.add_option("-t", "--template", dest="template", default="basic",
-                          help="Uses a custom template.")
-        parser.add_option("--force", dest="force", action="store_true",
-                          help="If the spider already exists, overwrite it with the template")
+        parser.add_argument("-l", "--list", dest="list", action="store_true",
+                            help="List available templates")
+        parser.add_argument("-e", "--edit", dest="edit", action="store_true",
+                            help="Edit spider after creating it")
+        parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
+                            help="Dump template to standard output")
+        parser.add_argument("-t", "--template", dest="template", default="basic",
+                            help="Uses a custom template.")
+        parser.add_argument("--force", dest="force", action="store_true",
+                            help="If the spider already exists, overwrite it with the template")
 
     def run(self, args, opts):
         if opts.list:
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 52118db1b..a3f6b96f4 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand):
 
     def add_options(self, parser):
         BaseRunSpiderCommand.add_options(self, parser)
-        parser.add_option("--spider", dest="spider", default=None,
-                          help="use this spider without looking for one")
-        parser.add_option("--pipelines", action="store_true",
-                          help="process items through pipelines")
-        parser.add_option("--nolinks", dest="nolinks", action="store_true",
-                          help="don't show links to follow (extracted requests)")
-        parser.add_option("--noitems", dest="noitems", action="store_true",
-                          help="don't show scraped items")
-        parser.add_option("--nocolour", dest="nocolour", action="store_true",
-                          help="avoid using pygments to colorize the output")
-        parser.add_option("-r", "--rules", dest="rules", action="store_true",
-                          help="use CrawlSpider rules to discover the callback")
-        parser.add_option("-c", "--callback", dest="callback",
-                          help="use this callback for parsing, instead looking for a callback")
-        parser.add_option("-m", "--meta", dest="meta",
-                          help="inject extra meta into the Request, it must be a valid raw json string")
-        parser.add_option("--cbkwargs", dest="cbkwargs",
-                          help="inject extra callback kwargs into the Request, it must be a valid raw json string")
-        parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
-                          help="maximum depth for parsing requests [default: %default]")
-        parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
-                          help="print each depth level one by one")
+        parser.add_argument("--spider", dest="spider", default=None,
+                            help="use this spider without looking for one")
+        parser.add_argument("--pipelines", action="store_true",
+                            help="process items through pipelines")
+        parser.add_argument("--nolinks", dest="nolinks", action="store_true",
+                            help="don't show links to follow (extracted requests)")
+        parser.add_argument("--noitems", dest="noitems", action="store_true",
+                            help="don't show scraped items")
+        parser.add_argument("--nocolour", dest="nocolour", action="store_true",
+                            help="avoid using pygments to colorize the output")
+        parser.add_argument("-r", "--rules", dest="rules", action="store_true",
+                            help="use CrawlSpider rules to discover the callback")
+        parser.add_argument("-c", "--callback", dest="callback",
+                            help="use this callback for parsing, instead looking for a callback")
+        parser.add_argument("-m", "--meta", dest="meta",
+                            help="inject extra meta into the Request, it must be a valid raw json string")
+        parser.add_argument("--cbkwargs", dest="cbkwargs",
+                            help="inject extra callback kwargs into the Request, it must be a valid raw json string")
+        parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
+                            help="maximum depth for parsing requests [default: %default]")
+        parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
+                            help="print each depth level one by one")
 
     @property
     def max_level(self):
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index 8d49e440f..1b2e2601e 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -18,16 +18,16 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("--get", dest="get", metavar="SETTING",
-                          help="print raw setting value")
-        parser.add_option("--getbool", dest="getbool", metavar="SETTING",
-                          help="print setting value, interpreted as a boolean")
-        parser.add_option("--getint", dest="getint", metavar="SETTING",
-                          help="print setting value, interpreted as an integer")
-        parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
-                          help="print setting value, interpreted as a float")
-        parser.add_option("--getlist", dest="getlist", metavar="SETTING",
-                          help="print setting value, interpreted as a list")
+        parser.add_argument("--get", dest="get", metavar="SETTING",
+                            help="print raw setting value")
+        parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
+                            help="print setting value, interpreted as a boolean")
+        parser.add_argument("--getint", dest="getint", metavar="SETTING",
+                            help="print setting value, interpreted as an integer")
+        parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
+                            help="print setting value, interpreted as a float")
+        parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
+                            help="print setting value, interpreted as a list")
 
     def run(self, args, opts):
         settings = self.crawler_process.settings
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index de81986d8..f67a5886a 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -33,12 +33,12 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("-c", dest="code",
-                          help="evaluate the code in the shell, print the result and exit")
-        parser.add_option("--spider", dest="spider",
-                          help="use this spider")
-        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
-                          help="do not handle HTTP 3xx status codes and print response as-is")
+        parser.add_argument("-c", dest="code",
+                            help="evaluate the code in the shell, print the result and exit")
+        parser.add_argument("--spider", dest="spider",
+                            help="use this spider")
+        parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
+                            help="do not handle HTTP 3xx status codes and print response as-is")
 
     def update_vars(self, vars):
         """You can use this function to update the Scrapy objects that will be
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 1237610cb..c6a3c273a 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -16,8 +16,8 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
-                          help="also display twisted/python/platform info (useful for bug reports)")
+        parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
+                            help="also display twisted/python/platform info (useful for bug reports)")
 
     def run(self, args, opts):
         if opts.verbose:
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index c8f873334..b1f52abe2 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -1,3 +1,4 @@
+import argparse
 from scrapy.commands import fetch
 from scrapy.utils.response import open_in_browser
 
@@ -12,7 +13,7 @@ class Command(fetch.Command):
 
     def add_options(self, parser):
         super().add_options(parser)
-        parser.remove_option("--headers")
+        parser.add_argument('--headers', help=argparse.SUPPRESS)
 
     def _print_response(self, response, opts):
         open_in_browser(response)
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index ed3848d88..f21ee971d 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,6 +1,9 @@
 import os
+import argparse
 from os.path import join, abspath, isfile, exists
 from twisted.internet import defer
+from scrapy.commands import parse
+from scrapy.settings import Settings
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
 from scrapy.utils.python import to_unicode
@@ -239,3 +242,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
         content = '[\n{},\n{"foo": "bar"}\n]'
         with open(file_path, 'r') as f:
             self.assertEqual(f.read(), content)
+
+    def test_parse_add_options(self):
+        command = parse.Command()
+        command.settings = Settings()
+        parser = argparse.ArgumentParser(
+            prog='scrapy', formatter_class=argparse.HelpFormatter,
+            conflict_handler='resolve', prefix_chars='-'
+        )
+        command.add_options(parser)
+        namespace = parser.parse_args(
+            ['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name]
+        )
+        self.assertTrue(namespace.nolinks)
+        self.assertEqual(namespace.depth, 2)
+        self.assertEqual(namespace.spider, self.spider_name)
+        self.assertTrue(namespace.verbose)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 7473b53df..7cd19b29a 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -1,6 +1,6 @@
 import inspect
 import json
-import optparse
+import argparse
 import os
 import platform
 import re
@@ -23,7 +23,7 @@ from twisted.python.versions import Version
 from twisted.trial import unittest
 
 import scrapy
-from scrapy.commands import ScrapyCommand
+from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter
 from scrapy.commands.startproject import IGNORE
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
@@ -37,19 +37,28 @@ class CommandSettings(unittest.TestCase):
     def setUp(self):
         self.command = ScrapyCommand()
         self.command.settings = Settings()
-        self.parser = optparse.OptionParser(
-            formatter=optparse.TitledHelpFormatter(),
-            conflict_handler='resolve',
-        )
+        self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
+                                              conflict_handler='resolve')
         self.command.add_options(self.parser)
 
     def test_settings_json_string(self):
         feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}'
-        opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])
+        opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])
         self.command.process_options(args, opts)
         self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings)
         self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json))
 
+    def test_help_formatter(self):
+        formatter = ScrapyHelpFormatter(prog='scrapy')
+        part_strings = ['usage: scrapy genspider [options] <name> <domain>\n\n',
+                        '\n', 'optional arguments:\n', '\n', 'Global Options:\n']
+        self.assertEqual(
+            formatter._join_parts(part_strings),
+            ('Usage\n=====\n  scrapy genspider [options] <name> <domain>\n\n\n'
+             'Optional Arguments\n==================\n\n'
+             'Global Options\n--------------\n')
+        )
+
 
 class ProjectTest(unittest.TestCase):
     project_name = 'testproject'
@@ -812,6 +821,21 @@ class BenchCommandTest(CommandTest):
         self.assertNotIn('Unhandled Error', log)
 
 
+class ViewCommandTest(CommandTest):
+
+    def test_methods(self):
+        command = view.Command()
+        command.settings = Settings()
+        parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-',
+                                         formatter_class=ScrapyHelpFormatter,
+                                         conflict_handler='resolve')
+        command.add_options(parser)
+        self.assertEqual(command.short_desc(),
+                         "Open URL in browser, as seen by Scrapy")
+        self.assertIn("URL using the Scrapy downloader and show its",
+                      command.long_desc())
+
+
 class CrawlCommandTest(CommandTest):
 
     def crawl(self, code, args=()):

From 5d7c0a5f861327c1a51ffcb3a39e283eb999fae7 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Thu, 10 Feb 2022 14:50:12 +0500
Subject: [PATCH 542/716] Use toscrape.com instead of example.com in
 test_command_check. (#5407)

---
 tests/test_command_check.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 34f5e59dd..c3d705194 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -19,11 +19,11 @@ import scrapy
 
 class CheckSpider(scrapy.Spider):
     name = '{self.spider_name}'
-    start_urls = ['http://example.com']
+    start_urls = ['http://toscrape.com']
 
     def parse(self, response, **cb_kwargs):
         \"\"\"
-        @url http://example.com
+        @url http://toscrape.com
         {contracts}
         \"\"\"
         {parse_def}

From befb6df119058db8a6a340b8235ccb565a60f3ca Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Fri, 11 Feb 2022 06:19:27 -0300
Subject: [PATCH 543/716] Remove Python 2 code from WrappedRequest

---
 scrapy/http/cookies.py     | 6 +-----
 tests/test_http_cookies.py | 1 -
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index bf4ae7b45..b43c383fe 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -142,10 +142,6 @@ class WrappedRequest:
         """
         return self.request.meta.get('is_unverifiable', False)
 
-    def get_origin_req_host(self):
-        return urlparse_cached(self.request).hostname
-
-    # python3 uses attributes instead of methods
     @property
     def full_url(self):
         return self.get_full_url()
@@ -164,7 +160,7 @@ class WrappedRequest:
 
     @property
     def origin_req_host(self):
-        return self.get_origin_req_host()
+        return urlparse_cached(self.request).hostname
 
     def has_header(self, name):
         return name in self.request.headers
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 540e27907..08420332c 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -34,7 +34,6 @@ class WrappedRequestTest(TestCase):
         self.assertTrue(self.wrapped.unverifiable)
 
     def test_get_origin_req_host(self):
-        self.assertEqual(self.wrapped.get_origin_req_host(), 'www.example.com')
         self.assertEqual(self.wrapped.origin_req_host, 'www.example.com')
 
     def test_has_header(self):

From bbb693d046a1942965ea9579bf7bb8a20fc92ba3 Mon Sep 17 00:00:00 2001
From: Boris Zabolotskikh <vaiho@bk.ru>
Date: Mon, 14 Feb 2022 12:07:45 +0300
Subject: [PATCH 544/716] Update downloader-middleware.rst

Added a link to the method
---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index caf44a903..44201d0d5 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -89,7 +89,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
       methods of installed middleware is always called on every response.
 
       If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling
-      process_request methods and reschedule the returned request. Once the newly returned
+      :meth:`process_request` methods and reschedule the returned request. Once the newly returned
       request is performed, the appropriate middleware chain will be called on
       the downloaded response.
 

From 187b5c887602218dee2fb57ad1dba223c11ce84f Mon Sep 17 00:00:00 2001
From: Abhishek K M <67158080+Sync271@users.noreply.github.com>
Date: Mon, 14 Feb 2022 23:46:53 +0530
Subject: [PATCH 545/716] Update the documentation link for robots.txt (#5415)

---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index caf44a903..f9208d550 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1019,7 +1019,7 @@ Parsers vary in several aspects:
   (shorter) rule
 
 Performance comparison of different parsers is available at `the following link
-<https://anubhavp28.github.io/gsoc-weekly-checkin-12/>`_.
+<https://github.com/scrapy/scrapy/issues/3969>`_.
 
 .. _protego-parser:
 

From 3b42ccfebadd72d9b455f6526ab63835b72b1558 Mon Sep 17 00:00:00 2001
From: Gowtham Chowdary <42214663+GowthamChowdary@users.noreply.github.com>
Date: Thu, 17 Feb 2022 02:03:56 +0530
Subject: [PATCH 546/716] Add a link to Discord (#5422)

---
 docs/index.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/index.rst b/docs/index.rst
index 433798aa8..69becd4a8 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -24,12 +24,14 @@ Having trouble? We'd like to help!
 * Search for questions on the archives of the `scrapy-users mailing list`_.
 * Ask a question in the `#scrapy IRC channel`_,
 * Report bugs with Scrapy in our `issue tracker`_.
+* Join the Discord community `Scrapy Discord`_.
 
 .. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
 .. _Scrapy subreddit: https://www.reddit.com/r/scrapy/
 .. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
 .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
 .. _issue tracker: https://github.com/scrapy/scrapy/issues
+.. _Scrapy Discord : https://discord.gg/mv3yErfpvq
 
 
 First steps

From 08557e09db4bcb109eb78e9058622ab5cef77415 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Wed, 23 Feb 2022 23:52:18 +0500
Subject: [PATCH 547/716] Pin old markupsafe when we pin old mitmproxy (#5427)

---
 tox.ini | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tox.ini b/tox.ini
index 2031a2d92..fcd3563b2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -17,6 +17,8 @@ deps =
     #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
     mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
     mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
+    # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
+    markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy'
     # Extras
     botocore>=1.4.87
 passenv =
@@ -127,6 +129,9 @@ deps =
     robotexclusionrulesparser
     Pillow>=4.0.0
     Twisted[http2]>=17.9.0
+    # Twisted[http2] currently forces old mitmproxy because of h2 version restrictions in their deps,
+    # so we need to pin old markupsafe here too
+    markupsafe < 2.1.0
 
 [testenv:asyncio]
 commands =

From aa0306a167ef34b23cc2ec407a48359a4b5a8d0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 1 Mar 2022 12:16:37 +0100
Subject: [PATCH 548/716] Cover 2.6.0 in the release notes (#5399)

---
 docs/conf.py                     |   4 +
 docs/index.rst                   |   4 +-
 docs/news.rst                    | 379 +++++++++++++++++++++++++++++--
 docs/topics/asyncio.rst          |  33 ++-
 docs/topics/commands.rst         |  10 +-
 docs/topics/feed-exports.rst     |  10 +-
 docs/topics/media-pipeline.rst   |   2 +
 docs/topics/request-response.rst |   4 +
 docs/topics/spiders.rst          |   5 +-
 scrapy/utils/defer.py            |   4 +-
 10 files changed, 407 insertions(+), 48 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index d5e139e66..55aa72d5a 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -303,10 +303,14 @@ intersphinx_mapping = {
 hoverxref_auto_ref = True
 hoverxref_role_types = {
     "class": "tooltip",
+    "command": "tooltip",
     "confval": "tooltip",
     "hoverxref": "tooltip",
     "mod": "tooltip",
     "ref": "tooltip",
+    "reqmeta": "tooltip",
+    "setting": "tooltip",
+    "signal": "tooltip",
 }
 hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
 
diff --git a/docs/index.rst b/docs/index.rst
index 69becd4a8..75e08f537 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -12,6 +12,8 @@ testing.
 .. _web crawling: https://en.wikipedia.org/wiki/Web_crawler
 .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
 
+.. _getting-help:
+
 Getting help
 ============
 
@@ -31,7 +33,7 @@ Having trouble? We'd like to help!
 .. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
 .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
 .. _issue tracker: https://github.com/scrapy/scrapy/issues
-.. _Scrapy Discord : https://discord.gg/mv3yErfpvq
+.. _Scrapy Discord: https://discord.gg/mv3yErfpvq
 
 
 First steps
diff --git a/docs/news.rst b/docs/news.rst
index 47a808693..2128f2f0e 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1,30 +1,360 @@
-.. note::
-    .. versionchanged:: VERSION
-
-    The Twisted reactor is now installed when
-    :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a
-    :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this,
-    :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now
-    honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions
-    they are silently ignored when set there and you need to set these settings
-    in some other way.
-
-
-.. note::
-    .. versionchanged:: VERSION
-
-    Previously this setting had no effect in a spider
-    :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but
-    if you :ref:`run several spiders in one process <run-multiple-spiders>`,
-    they must not have different values for this setting, because they will use
-    a single reactor instance.
-
-
 .. _news:
 
 Release notes
 =============
 
+.. _release-2.6.0:
+
+Scrapy 2.6.0 (2022-02-??)
+-------------------------
+
+Highlights:
+
+*   Python 3.10 support
+
+*   :ref:`asyncio support <using-asyncio>` is no longer considered
+    experimental, and works out-of-the-box on Windows regardless of your Python
+    version
+
+*   Feed exports now support :class:`pathlib.Path` output paths and per-feed
+    :ref:`item filtering <item-filter>` and
+    :ref:`post-processing <post-processing>`
+
+Modified requirements
+~~~~~~~~~~~~~~~~~~~~~
+
+-   The h2_ dependency is now optional, only needed to
+    :ref:`enable HTTP/2 support <http2>`. (:issue:`5113`)
+
+    .. _h2: https://pypi.org/project/h2/
+
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+-   The ``formdata`` parameter of :class:`~scrapy.FormRequest`, if specified
+    for a non-POST request, now overrides the URL query string, instead of
+    being appended to it. (:issue:`2919`, :issue:`3579`)
+
+-   When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, now
+    the return value of that function, and not the ``params`` input parameter,
+    will determine the feed URI parameters, unless that return value is
+    ``None``. (:issue:`4962`, :issue:`4966`)
+
+-   In :class:`scrapy.core.engine.ExecutionEngine`, methods
+    :meth:`~scrapy.core.engine.ExecutionEngine.crawl`,
+    :meth:`~scrapy.core.engine.ExecutionEngine.download`,
+    :meth:`~scrapy.core.engine.ExecutionEngine.schedule`,
+    and :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`
+    now raise :exc:`RuntimeError` if called before
+    :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`. (:issue:`5090`)
+
+    These methods used to assume that
+    :attr:`ExecutionEngine.slot <scrapy.core.engine.ExecutionEngine.slot>` had
+    been defined by a prior call to
+    :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`, so they were
+    raising :exc:`AttributeError` instead.
+
+-   If the API of the configured :ref:`scheduler <topics-scheduler>` does not
+    meet expectations, :exc:`TypeError` is now raised at startup time. Before,
+    other exceptions would be raised at run time. (:issue:`3559`)
+
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+-   ``scrapy.http.TextResponse.body_as_unicode``, deprecated in Scrapy 2.2, has
+    now been removed. (:issue:`5393`)
+
+-   ``scrapy.item.BaseItem``, deprecated in Scrapy 2.2, has now been removed.
+    (:issue:`5398`)
+
+-   ``scrapy.item.DictItem``, deprecated in Scrapy 1.8, has now been removed.
+    (:issue:`5398`)
+
+-   ``scrapy.Spider.make_requests_from_url``, deprecated in Scrapy 1.4, has now
+    been removed. (:issue:`4178`, :issue:`4356`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+-   When a function is assigned to the :setting:`FEED_URI_PARAMS` setting,
+    returning ``None`` or modifying the ``params`` input parameter is now
+    deprecated. Return a new dictionary instead. (:issue:`4962`, :issue:`4966`)
+
+-   :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`)
+
+    -   Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new
+        :meth:`Request.to_dict <scrapy.http.Request.to_dict>` method.
+
+    -   Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new
+        :func:`scrapy.utils.request.request_from_dict` function.
+
+-   In :mod:`scrapy.squeues`, the following queue classes are deprecated:
+    :class:`~scrapy.squeues.PickleFifoDiskQueueNonRequest`,
+    :class:`~scrapy.squeues.PickleLifoDiskQueueNonRequest`,
+    :class:`~scrapy.squeues.MarshalFifoDiskQueueNonRequest`,
+    and :class:`~scrapy.squeues.MarshalLifoDiskQueueNonRequest`. You should
+    instead use:
+    :class:`~scrapy.squeues.PickleFifoDiskQueue`,
+    :class:`~scrapy.squeues.PickleLifoDiskQueue`,
+    :class:`~scrapy.squeues.MarshalFifoDiskQueue`,
+    and :class:`~scrapy.squeues.MarshalLifoDiskQueue`. (:issue:`5117`)
+
+-   Many aspects of :class:`scrapy.core.engine.ExecutionEngine` that come from
+    a time when this class could handle multiple :class:`~scrapy.Spider`
+    objects at a time have been deprecated. (:issue:`5090`)
+
+    -   The :meth:`~scrapy.core.engine.ExecutionEngine.has_capacity` method
+        is deprecated.
+
+    -   The :meth:`~scrapy.core.engine.ExecutionEngine.schedule` method is
+        deprecated, use :meth:`~scrapy.core.engine.ExecutionEngine.crawl` or
+        :meth:`~scrapy.core.engine.ExecutionEngine.download` instead.
+
+    -   The :attr:`~scrapy.core.engine.ExecutionEngine.open_spiders` attribute
+        is deprecated, use :attr:`~scrapy.core.engine.ExecutionEngine.spider`
+        instead.
+
+    -   The ``spider`` parameter is deprecated for the following methods:
+
+        -   :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`
+
+        -   :meth:`~scrapy.core.engine.ExecutionEngine.crawl`
+
+        -   :meth:`~scrapy.core.engine.ExecutionEngine.download`
+
+        Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`
+        first to set the :class:`~scrapy.Spider` object.
+
+
+New features
+~~~~~~~~~~~~
+
+-   You can now use :ref:`item filtering <item-filter>` to control which items
+    are exported to each output feed. (:issue:`4575`, :issue:`5178`,
+    :issue:`5161`, :issue:`5203`)
+
+-   You can now apply :ref:`post-processing <post-processing>` to feeds, and
+    :ref:`built-in post-processing plugins <builtin-plugins>` are provided for
+    output file compression. (:issue:`2174`, :issue:`5168`, :issue:`5190`)
+
+-   The :setting:`FEEDS` setting now supports :class:`pathlib.Path` objects as
+    keys. (:issue:`5383`, :issue:`5384`)
+
+-   Enabling :ref:`asyncio <using-asyncio>` while using Windows and Python 3.8
+    or later will automatically switch the asyncio event loop to one that
+    allows Scrapy to work. See :ref:`asyncio-windows`. (:issue:`4976`,
+    :issue:`5315`)
+
+-   The :command:`genspider` command now supports a start URL instead of a
+    domain name. (:issue:`4439`)
+
+-   :mod:`scrapy.utils.defer` gained 2 new functions,
+    :func:`~scrapy.utils.defer.deferred_to_future` and
+    :func:`~scrapy.utils.defer.maybe_deferred_to_future`, to help :ref:`await
+    on Deferreds when using the asyncio reactor <asyncio-await-dfd>`.
+    (:issue:`5288`)
+
+-   :ref:`Amazon S3 feed export storage <topics-feed-storage-s3>` gained
+    support for `temporary security credentials`_
+    (:setting:`AWS_SESSION_TOKEN`) and endpoint customization
+    (:setting:`AWS_ENDPOINT_URL`). (:issue:`4998`, :issue:`5210`)
+
+    .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
+
+-   New :setting:`LOG_FILE_APPEND` setting to allow truncating the log file.
+    (:issue:`5279`)
+
+-   :attr:`Request.cookies <scrapy.Request.cookies>` values that are
+    :class:`bool`, :class:`float` or :class:`int` are cast to :class:`str`.
+    (:issue:`5252`, :issue:`5253`)
+
+-   You may now raise :exc:`~scrapy.exceptions.CloseSpider` from a handler of
+    the :signal:`spider_idle` signal to customize the reason why the spider is
+    stopping. (:issue:`5191`)
+
+-   When using
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`, the
+    proxy URL for non-HTTPS HTTP/1.1 requests no longer needs to include a URL
+    scheme. (:issue:`4505`, :issue:`4649`)
+
+-   All built-in queues now expose a ``peek`` method that returns the next
+    queue object (like ``pop``) but does not remove the returned object from
+    the queue. (:issue:`5112`)
+
+    If the underlying queue does not support peeking (e.g. because you are not
+    using ``queuelib`` 1.6.1 or later), the ``peek`` method raises
+    :exc:`NotImplementedError`.
+
+-   :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have
+    an ``attributes`` attribute that makes subclassing easier. For
+    :class:`~scrapy.http.Request`, it also allows subclasses to work with
+    :func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`,
+    :issue:`5130`, :issue:`5218`)
+
+-   The :meth:`~scrapy.core.scheduler.BaseScheduler.open` and
+    :meth:`~scrapy.core.scheduler.BaseScheduler.close` methods of the
+    :ref:`scheduler <topics-scheduler>` are now optional. (:issue:`3559`)
+
+-   HTTP/1.1 :exc:`~scrapy.core.downloader.handlers.http11.TunnelError`
+    exceptions now only truncate response bodies longer than 1000 characters,
+    instead of those longer than 32 characters, making it easier to debug such
+    errors. (:issue:`4881`, :issue:`5007`)
+
+-   :class:`~scrapy.loader.ItemLoader` now supports non-text responses.
+    (:issue:`5145`, :issue:`5269`)
+
+
+Bug fixes
+~~~~~~~~~
+
+-   The :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` settings
+    are no longer ignored if defined in :attr:`~scrapy.Spider.custom_settings`.
+    (:issue:`4485`, :issue:`5352`)
+
+-   Removed a module-level Twisted reactor import that could prevent
+    :ref:`using the asyncio reactor <using-asyncio>`. (:issue:`5357`)
+
+-   The :command:`startproject` command works with existing folders again.
+    (:issue:`4665`, :issue:`4676`)
+
+-   The :setting:`FEED_URI_PARAMS` setting now behaves as documented.
+    (:issue:`4962`, :issue:`4966`)
+
+-   :attr:`Request.cb_kwargs <scrapy.Request.cb_kwargs>` once again allows the
+    ``callback`` keyword. (:issue:`5237`, :issue:`5251`, :issue:`5264`)
+
+-   Made :func:`scrapy.utils.response.open_in_browser` support more complex
+    HTML. (:issue:`5319`, :issue:`5320`)
+
+-   Fixed :attr:`CSVFeedSpider.quotechar
+    <scrapy.spiders.CSVFeedSpider.quotechar>` being interpreted as the CSV file
+    encoding. (:issue:`5391`, :issue:`5394`)
+
+-   Added missing setuptools_ to the list of dependencies. (:issue:`5122`)
+
+    .. _setuptools: https://pypi.org/project/setuptools/
+
+-   :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
+    now also works as expected with links that have comma-separated ``rel``
+    attribute values including ``nofollow``. (:issue:`5225`)
+
+-   Fixed a :exc:`TypeError` that could be raised during :ref:`feed export
+    <topics-feed-exports>` parameter parsing. (:issue:`5359`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+-   :ref:`asyncio support <using-asyncio>` is no longer considered
+    experimental. (:issue:`5332`)
+
+-   Included :ref:`Windows-specific help for asyncio usage <asyncio-windows>`.
+    (:issue:`4976`, :issue:`5315`)
+
+-   Rewrote :ref:`topics-headless-browsing` with up-to-date best practices.
+    (:issue:`4484`, :issue:`4613`)
+
+-   Documented :ref:`local file naming in media pipelines
+    <topics-file-naming>`. (:issue:`5069`, :issue:`5152`)
+
+-   :ref:`faq` now covers spider file name collision issues. (:issue:`2680`,
+    :issue:`3669`)
+
+-   Provided better context and instructions to disable the
+    :setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`)
+
+-   Documented that :ref:`reppy-parser` does not support Python 3.9+.
+    (:issue:`5226`, :issue:`5231`)
+
+-   Documented :ref:`the scheduler component <topics-scheduler>`.
+    (:issue:`3537`, :issue:`3559`)
+
+-   Documented the method used by :ref:`media pipelines
+    <topics-media-pipeline>` to :ref:`determine if a file has expired
+    <file-expiration>`. (:issue:`5120`, :issue:`5254`)
+
+-   :ref:`run-multiple-spiders` now features
+    :func:`scrapy.utils.project.get_project_settings` usage. (:issue:`5070`)
+
+-   :ref:`run-multiple-spiders` now covers what happens when you define
+    different per-spider values for some settings that cannot differ at run
+    time. (:issue:`4485`, :issue:`5352`)
+
+-   Extended the documentation of the
+    :class:`~scrapy.extensions.statsmailer.StatsMailer` extension.
+    (:issue:`5199`, :issue:`5217`)
+
+-   Added :setting:`JOBDIR` to :ref:`topics-settings`. (:issue:`5173`,
+    :issue:`5224`)
+
+-   Documented :attr:`Spider.attribute <scrapy.Spider.attribute>`.
+    (:issue:`5174`, :issue:`5244`)
+
+-   Documented :attr:`TextResponse.urljoin <scrapy.http.TextResponse.urljoin>`.
+    (:issue:`1582`)
+
+-   Added the ``body_length`` parameter to the documented signature of the
+    :signal:`headers_received` signal. (:issue:`5270`)
+
+-   Clarified :meth:`SelectorList.get <scrapy.selector.SelectorList.get>` usage
+    in the :ref:`tutorial <intro-tutorial>`. (:issue:`5256`)
+
+-   The documentation now features the shortest import path of classes with
+    multiple import paths. (:issue:`2733`, :issue:`5099`)
+
+-   ``quotes.toscrape.com`` references now use HTTPS instead of HTTP.
+    (:issue:`5395`, :issue:`5396`)
+
+-   Added a link to `our Discord server <https://discord.gg/mv3yErfpvq>`_
+    to :ref:`getting-help`. (:issue:`5421`, :issue:`5422`)
+
+-   The pronunciation of the project name is now :ref:`officially
+    <intro-overview>` /ˈskreɪpaɪ/. (:issue:`5280`, :issue:`5281`)
+
+-   Added the Scrapy logo to the README. (:issue:`5255`, :issue:`5258`)
+
+-   Fixed issues and implemented minor improvements. (:issue:`3155`,
+    :issue:`4335`, :issue:`5074`, :issue:`5098`, :issue:`5134`, :issue:`5180`,
+    :issue:`5194`, :issue:`5239`, :issue:`5266`, :issue:`5271`, :issue:`5273`,
+    :issue:`5274`, :issue:`5276`, :issue:`5347`, :issue:`5356`, :issue:`5414`,
+    :issue:`5415`, :issue:`5416`, :issue:`5419`, :issue:`5420`)
+
+
+Quality Assurance
+~~~~~~~~~~~~~~~~~
+
+-   Added support for Python 3.10. (:issue:`5212`, :issue:`5221`,
+    :issue:`5265`)
+
+-   Significantly reduced memory usage by
+    :func:`scrapy.utils.response.response_httprepr`, used by the
+    :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` downloader
+    middleware, which is enabled by default. (:issue:`4964`, :issue:`4972`)
+
+-   Removed uses of the deprecated :mod:`optparse` module. (:issue:`5366`,
+    :issue:`5374`)
+
+-   Extended typing hints. (:issue:`5077`, :issue:`5090`, :issue:`5100`,
+    :issue:`5108`, :issue:`5171`, :issue:`5215`, :issue:`5334`)
+
+-   Improved tests, fixed CI issues, removed unused code. (:issue:`5094`,
+    :issue:`5157`, :issue:`5162`, :issue:`5198`, :issue:`5207`, :issue:`5208`,
+    :issue:`5229`, :issue:`5298`, :issue:`5299`, :issue:`5310`, :issue:`5316`,
+    :issue:`5333`, :issue:`5388`, :issue:`5389`, :issue:`5400`, :issue:`5401`,
+    :issue:`5404`, :issue:`5405`, :issue:`5407`, :issue:`5410`, :issue:`5412`,
+    :issue:`5425`, :issue:`5427`)
+
+-   Implemented improvements for contributors. (:issue:`5080`, :issue:`5082`,
+    :issue:`5177`, :issue:`5200`)
+
+-   Implemented cleanups. (:issue:`5095`, :issue:`5106`, :issue:`5209`,
+    :issue:`5228`, :issue:`5235`, :issue:`5245`, :issue:`5246`, :issue:`5292`,
+    :issue:`5314`, :issue:`5322`)
+
+
 .. _release-2.5.1:
 
 Scrapy 2.5.1 (2021-10-05)
@@ -1000,9 +1330,8 @@ Bug fixes
 *   zope.interface 5.0.0 and later versions are now supported
     (:issue:`4447`, :issue:`4448`)
 
-*   :meth:`Spider.make_requests_from_url
-    <scrapy.spiders.Spider.make_requests_from_url>`, deprecated in Scrapy
-    1.4.0, now issues a warning when used (:issue:`4412`)
+*   ``Spider.make_requests_from_url``, deprecated in Scrapy 1.4.0, now issues a
+    warning when used (:issue:`4412`)
 
 
 Documentation
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 8712d4268..3a6941a2c 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -10,6 +10,7 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
 asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
 :mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
 
+
 .. _install-asyncio:
 
 Installing the asyncio reactor
@@ -25,6 +26,7 @@ reactor manually. You can do that using
 
     install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
 
+
 .. _using-custom-loops:
 
 Using custom asyncio loops
@@ -34,20 +36,30 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the
 :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
 use it instead of the default asyncio event loop.
 
-.. _asyncio-await-dfd:
+
+.. _asyncio-windows:
 
 Windows-specific notes
 ======================
 
 The Windows implementation of :mod:`asyncio` can use two event loop
-implementations: :class:`~asyncio.SelectorEventLoop` (default before Python
-3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop`
-(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the
-event loop class needs to be changed. Scrapy since VERSION does this
-automatically when you change the :setting:`TWISTED_REACTOR` setting or call
-:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor
-by other means or use an older Scrapy version you need to call the following
-code before installing the reactor::
+implementations:
+
+-   :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required
+    when using Twisted.
+
+-   :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work
+    with Twisted.
+
+So on Python 3.8+ the event loop class needs to be changed.
+
+.. versionchanged:: 2.6.0
+   The event loop class is changed automatically when you change the
+   :setting:`TWISTED_REACTOR` setting or call
+   :func:`~scrapy.utils.reactor.install_reactor`.
+
+To change the event loop class manually, call the following code before
+installing the reactor::
 
     import asyncio
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
@@ -64,6 +76,9 @@ yourself, or in some code that runs before the reactor is installed, e.g.
 
 .. _playwright: https://github.com/microsoft/playwright-python
 
+
+.. _asyncio-await-dfd:
+
 Awaiting on Deferreds
 =====================
 
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index eef6b36ff..8c0b8e55f 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -230,10 +230,16 @@ Usage example::
 genspider
 ---------
 
-* Syntax: ``scrapy genspider [-t template] <name> <domain>``
+* Syntax: ``scrapy genspider [-t template] <name> <domain or URL>``
 * Requires project: *no*
 
-Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
+.. versionadded:: 2.6.0
+   The ability to pass a URL instead of a domain.
+
+Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
+
+.. note:: Even if an HTTPS URL is specified, the protocol used in
+          ``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.
 
 Usage example::
 
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 7994027d2..9a13eb82f 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -278,7 +278,7 @@ feed URI, allowing item delivery to start way before the end of the crawl.
 Item filtering
 ==============
 
-.. versionadded:: VERSION
+.. versionadded:: 2.6.0
 
 You can filter items that you want to allow for a particular feed by using the
 ``item_classes`` option in :ref:`feeds options <feed-options>`. Only items of
@@ -318,7 +318,7 @@ ItemFilter
 Post-Processing
 ===============
 
-.. versionadded:: VERSION
+.. versionadded:: 2.6.0
 
 Scrapy provides an option to activate plugins to post-process feeds before they are exported
 to feed storages. In addition to using :ref:`builtin plugins <builtin-plugins>`, you
@@ -457,13 +457,13 @@ as a fallback value if that key is not provided for a specific feed definition:
 
     If undefined or empty, all items are exported.
 
-    .. versionadded:: VERSION
+    .. versionadded:: 2.6.0
 
 -   ``item_filter``: a :ref:`filter class <item-filter>` to filter items to export.
 
     :class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.
 
-    .. versionadded:: VERSION
+    .. versionadded:: 2.6.0
 
 -   ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
 
@@ -499,7 +499,7 @@ as a fallback value if that key is not provided for a specific feed definition:
 
     The plugins will be used in the order of the list passed.
 
-    .. versionadded:: VERSION
+    .. versionadded:: 2.6.0
 
 .. setting:: FEED_EXPORT_ENCODING
 
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 10d2ac990..7dff78390 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -356,6 +356,8 @@ setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
 Additional features
 ===================
 
+.. _file-expiration:
+
 File expiration
 ---------------
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index e0435e901..92a471faf 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -110,6 +110,10 @@ Request objects
             :class:`Request.cookies <scrapy.Request>` parameter. This is a known
             current limitation that is being worked on.
 
+        .. versionadded:: 2.6.0
+           Cookie values that are :class:`bool`, :class:`float` or :class:`int`
+           are casted to :class:`str`.
+
     :type cookies: dict or list
 
     :param encoding: the encoding of this request (defaults to ``'utf-8'``).
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 99e74233a..ece02ae47 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -42,15 +42,12 @@ Even though this cycle applies (more or less) to any kind of spider, there are
 different kinds of default spiders bundled into Scrapy for different purposes.
 We will talk about those types here.
 
-.. module:: scrapy.spiders
-   :synopsis: Spiders base class, spider manager and spider middleware
-
 .. _topics-spiders-ref:
 
 scrapy.Spider
 =============
 
-.. class:: scrapy.spiders.Spider()
+.. class:: scrapy.spiders.Spider
 .. class:: scrapy.Spider()
 
    This is the simplest spider, and the one from which every other spider
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index d7adc0a77..7ecb8ea3f 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -182,7 +182,7 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
 
 def deferred_to_future(d: Deferred) -> Future:
     """
-    .. versionadded:: VERSION
+    .. versionadded:: 2.6.0
 
     Return an :class:`asyncio.Future` object that wraps *d*.
 
@@ -203,7 +203,7 @@ def deferred_to_future(d: Deferred) -> Future:
 
 def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
     """
-    .. versionadded:: VERSION
+    .. versionadded:: 2.6.0
 
     Return *d* as an object that can be awaited from a :ref:`Scrapy callable
     defined as a coroutine <coroutine-support>`.

From 8ce01b3b76d4634f55067d6cfdf632ec70ba304a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 1 Mar 2022 12:26:05 +0100
Subject: [PATCH 549/716] Merge pull request from GHSA-cjvr-mfj7-j4j8

* Do not carry over cookies to a different domain on redirect

* Cover the cookie-domain redirect fix in the release notes

* Cover 1.8.2 in the release notes

* Fix redirect Cookie handling when the cookie middleware is disabled

* Update the 1.8.2 release date
---
 docs/news.rst                              |  67 ++++++++-
 scrapy/downloadermiddlewares/redirect.py   |  31 ++++-
 tests/test_downloadermiddleware_cookies.py | 155 +++++++++++++++++++++
 3 files changed, 247 insertions(+), 6 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 2128f2f0e..aef12d9db 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5,11 +5,13 @@ Release notes
 
 .. _release-2.6.0:
 
-Scrapy 2.6.0 (2022-02-??)
+Scrapy 2.6.0 (2022-03-01)
 -------------------------
 
 Highlights:
 
+*   :ref:`Security fixes for cookie handling <2.6-security-fixes>`
+
 *   Python 3.10 support
 
 *   :ref:`asyncio support <using-asyncio>` is no longer considered
@@ -20,6 +22,37 @@ Highlights:
     :ref:`item filtering <item-filter>` and
     :ref:`post-processing <post-processing>`
 
+.. _2.6-security-fixes:
+
+Security bug fixes
+~~~~~~~~~~~~~~~~~~
+
+-   When a :class:`~scrapy.http.Request` object with cookies defined gets a
+    redirect response causing a new :class:`~scrapy.http.Request` object to be
+    scheduled, the cookies defined in the original
+    :class:`~scrapy.http.Request` object are no longer copied into the new
+    :class:`~scrapy.http.Request` object.
+
+    If you manually set the ``Cookie`` header on a
+    :class:`~scrapy.http.Request` object and the domain name of the redirect
+    URL is not an exact match for the domain of the URL of the original
+    :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped
+    from the new :class:`~scrapy.http.Request` object.
+
+    The old behavior could be exploited by an attacker to gain access to your
+    cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more
+    information.
+
+    .. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8
+
+    .. note:: It is still possible to enable the sharing of cookies between
+              different domains with a shared domain suffix (e.g.
+              ``example.com`` and any subdomain) by defining the shared domain
+              suffix (e.g. ``example.com``) as the cookie domain when defining
+              your cookies. See the documentation of the
+              :class:`~scrapy.http.Request` class for more information.
+
+
 Modified requirements
 ~~~~~~~~~~~~~~~~~~~~~
 
@@ -1842,6 +1875,38 @@ affect subclasses:
 
 (:issue:`3884`)
 
+.. _release-1.8.2:
+
+Scrapy 1.8.2 (2022-03-01)
+-------------------------
+
+**Security bug fixes:**
+
+-   When a :class:`~scrapy.http.Request` object with cookies defined gets a
+    redirect response causing a new :class:`~scrapy.http.Request` object to be
+    scheduled, the cookies defined in the original
+    :class:`~scrapy.http.Request` object are no longer copied into the new
+    :class:`~scrapy.http.Request` object.
+
+    If you manually set the ``Cookie`` header on a
+    :class:`~scrapy.http.Request` object and the domain name of the redirect
+    URL is not an exact match for the domain of the URL of the original
+    :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped
+    from the new :class:`~scrapy.http.Request` object.
+
+    The old behavior could be exploited by an attacker to gain access to your
+    cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more
+    information.
+
+    .. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8
+
+    .. note:: It is still possible to enable the sharing of cookies between
+              different domains with a shared domain suffix (e.g.
+              ``example.com`` and any subdomain) by defining the shared domain
+              suffix (e.g. ``example.com``) as the cookie domain when defining
+              your cookies. See the documentation of the
+              :class:`~scrapy.http.Request` class for more information.
+
 
 .. _release-1.8.1:
 
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 4053fecc5..fcd6c298b 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -4,6 +4,7 @@ from urllib.parse import urljoin, urlparse
 from w3lib.url import safe_url_string
 
 from scrapy.http import HtmlResponse
+from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.response import get_meta_refresh
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 
@@ -11,6 +12,21 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
 logger = logging.getLogger(__name__)
 
 
+def _build_redirect_request(source_request, *, url, method=None, body=None):
+    redirect_request = source_request.replace(
+        url=url,
+        method=method,
+        body=body,
+        cookies=None,
+    )
+    if 'Cookie' in redirect_request.headers:
+        source_request_netloc = urlparse_cached(source_request).netloc
+        redirect_request_netloc = urlparse_cached(redirect_request).netloc
+        if source_request_netloc != redirect_request_netloc:
+            del redirect_request.headers['Cookie']
+    return redirect_request
+
+
 class BaseRedirectMiddleware:
 
     enabled_setting = 'REDIRECT_ENABLED'
@@ -47,10 +63,15 @@ class BaseRedirectMiddleware:
             raise IgnoreRequest("max redirections reached")
 
     def _redirect_request_using_get(self, request, redirect_url):
-        redirected = request.replace(url=redirect_url, method='GET', body='')
-        redirected.headers.pop('Content-Type', None)
-        redirected.headers.pop('Content-Length', None)
-        return redirected
+        redirect_request = _build_redirect_request(
+            request,
+            url=redirect_url,
+            method='GET',
+            body='',
+        )
+        redirect_request.headers.pop('Content-Type', None)
+        redirect_request.headers.pop('Content-Length', None)
+        return redirect_request
 
 
 class RedirectMiddleware(BaseRedirectMiddleware):
@@ -80,7 +101,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
         redirected_url = urljoin(request.url, location)
 
         if response.status in (301, 307, 308) or request.method == 'HEAD':
-            redirected = request.replace(url=redirected_url)
+            redirected = _build_redirect_request(request, url=redirected_url)
             return self._redirect(redirected, request, spider, response.status)
 
         redirected = self._redirect_request_using_get(request, redirected_url)
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 36021bfbf..1747f3b94 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -6,8 +6,10 @@ import pytest
 
 from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
+from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, Request
+from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
@@ -23,9 +25,11 @@ class CookiesMiddlewareTest(TestCase):
     def setUp(self):
         self.spider = Spider('foo')
         self.mw = CookiesMiddleware()
+        self.redirect_middleware = RedirectMiddleware(settings=Settings())
 
     def tearDown(self):
         del self.mw
+        del self.redirect_middleware
 
     def test_basic(self):
         req = Request('http://scrapytest.org/')
@@ -368,3 +372,154 @@ class CookiesMiddlewareTest(TestCase):
         req4 = Request('http://example.org', cookies={'a': 'b'})
         assert self.mw.process_request(req4, self.spider) is None
         self.assertCookieValEqual(req4.headers['Cookie'], b'a=b')
+
+    def _test_cookie_redirect(
+        self,
+        source,
+        target,
+        *,
+        cookies1,
+        cookies2,
+    ):
+        input_cookies = {'a': 'b'}
+
+        if not isinstance(source, dict):
+            source = {'url': source}
+        if not isinstance(target, dict):
+            target = {'url': target}
+        target.setdefault('status', 301)
+
+        request1 = Request(cookies=input_cookies, **source)
+        self.mw.process_request(request1, self.spider)
+        cookies = request1.headers.get('Cookie')
+        self.assertEqual(cookies, b"a=b" if cookies1 else None)
+
+        response = Response(
+            headers={
+                'Location': target['url'],
+            },
+            **target,
+        )
+        self.assertEqual(
+            self.mw.process_response(request1, response, self.spider),
+            response,
+        )
+
+        request2 = self.redirect_middleware.process_response(
+            request1,
+            response,
+            self.spider,
+        )
+        self.assertIsInstance(request2, Request)
+
+        self.mw.process_request(request2, self.spider)
+        cookies = request2.headers.get('Cookie')
+        self.assertEqual(cookies, b"a=b" if cookies2 else None)
+
+    def test_cookie_redirect_same_domain(self):
+        self._test_cookie_redirect(
+            'https://toscrape.com',
+            'https://toscrape.com',
+            cookies1=True,
+            cookies2=True,
+        )
+
+    def test_cookie_redirect_same_domain_forcing_get(self):
+        self._test_cookie_redirect(
+            'https://toscrape.com',
+            {'url': 'https://toscrape.com', 'status': 302},
+            cookies1=True,
+            cookies2=True,
+        )
+
+    def test_cookie_redirect_different_domain(self):
+        self._test_cookie_redirect(
+            'https://toscrape.com',
+            'https://example.com',
+            cookies1=True,
+            cookies2=False,
+        )
+
+    def test_cookie_redirect_different_domain_forcing_get(self):
+        self._test_cookie_redirect(
+            'https://toscrape.com',
+            {'url': 'https://example.com', 'status': 302},
+            cookies1=True,
+            cookies2=False,
+        )
+
+    def _test_cookie_header_redirect(
+        self,
+        source,
+        target,
+        *,
+        cookies2,
+    ):
+        """Test the handling of a user-defined Cookie header when building a
+        redirect follow-up request.
+
+        We follow RFC 6265 for cookie handling. The Cookie header can only
+        contain a list of key-value pairs (i.e. no additional cookie
+        parameters like Domain or Path). Because of that, we follow the same
+        rules that we would follow for the handling of the Set-Cookie response
+        header when the Domain is not set: the cookies must be limited to the
+        target URL domain (not even subdomains can receive those cookies).
+
+        .. note:: This method tests the scenario where the cookie middleware is
+                  disabled. Because of known issue #1992, when the cookies
+                  middleware is enabled we do not need to be concerned about
+                  the Cookie header getting leaked to unintended domains,
+                  because the middleware empties the header from every request.
+        """
+        if not isinstance(source, dict):
+            source = {'url': source}
+        if not isinstance(target, dict):
+            target = {'url': target}
+        target.setdefault('status', 301)
+
+        request1 = Request(headers={'Cookie': b'a=b'}, **source)
+
+        response = Response(
+            headers={
+                'Location': target['url'],
+            },
+            **target,
+        )
+
+        request2 = self.redirect_middleware.process_response(
+            request1,
+            response,
+            self.spider,
+        )
+        self.assertIsInstance(request2, Request)
+
+        cookies = request2.headers.get('Cookie')
+        self.assertEqual(cookies, b"a=b" if cookies2 else None)
+
+    def test_cookie_header_redirect_same_domain(self):
+        self._test_cookie_header_redirect(
+            'https://toscrape.com',
+            'https://toscrape.com',
+            cookies2=True,
+        )
+
+    def test_cookie_header_redirect_same_domain_forcing_get(self):
+        self._test_cookie_header_redirect(
+            'https://toscrape.com',
+            {'url': 'https://toscrape.com', 'status': 302},
+            cookies2=True,
+        )
+
+    def test_cookie_header_redirect_different_domain(self):
+        self._test_cookie_header_redirect(
+            'https://toscrape.com',
+            'https://example.com',
+            cookies2=False,
+        )
+
+    def test_cookie_header_redirect_different_domain_forcing_get(self):
+        self._test_cookie_header_redirect(
+            'https://toscrape.com',
+            {'url': 'https://example.com', 'status': 302},
+            cookies2=False,
+        )

From e865c4430e58a4faa0e0766b23830f8423d6167a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 1 Mar 2022 12:38:19 +0100
Subject: [PATCH 550/716] Merge pull request from GHSA-mfjm-vh54-3f96

* Ignore cookies with a public suffix as domain unless it matches the request domain

* Fix the merge of 1.8.2 release notes

* Re-apply removal of tldextract restriction
---
 docs/news.rst                              |  24 +++
 scrapy/downloadermiddlewares/cookies.py    |  34 ++++-
 setup.py                                   |   1 +
 tests/test_downloadermiddleware_cookies.py | 170 +++++++++++++++++++++
 4 files changed, 226 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index aef12d9db..9590fb1c4 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -52,6 +52,18 @@ Security bug fixes
               your cookies. See the documentation of the
               :class:`~scrapy.http.Request` class for more information.
 
+-   When the domain of a cookie, either received in the ``Set-Cookie`` header
+    of a response or defined in a :class:`~scrapy.http.Request` object, is set
+    to a `public suffix <https://publicsuffix.org/>`_, the cookie is now
+    ignored unless the cookie domain is the same as the request domain.
+
+    The old behavior could be exploited by an attacker to inject cookies from a
+    controlled domain into your cookiejar that could be sent to other domains
+    not controlled by the attacker. Please, see the `mfjm-vh54-3f96 security
+    advisory`_ for more information.
+
+    .. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96
+
 
 Modified requirements
 ~~~~~~~~~~~~~~~~~~~~~
@@ -1875,6 +1887,7 @@ affect subclasses:
 
 (:issue:`3884`)
 
+
 .. _release-1.8.2:
 
 Scrapy 1.8.2 (2022-03-01)
@@ -1907,6 +1920,17 @@ Scrapy 1.8.2 (2022-03-01)
               your cookies. See the documentation of the
               :class:`~scrapy.http.Request` class for more information.
 
+-   When the domain of a cookie, either received in the ``Set-Cookie`` header
+    of a response or defined in a :class:`~scrapy.http.Request` object, is set
+    to a `public suffix <https://publicsuffix.org/>`_, the cookie is now
+    ignored unless the cookie domain is the same as the request domain.
+
+    The old behavior could be exploited by an attacker to inject cookies into
+    your requests to some other domains. Please, see the `mfjm-vh54-3f96
+    security advisory`_ for more information.
+
+    .. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96
+
 
 .. _release-1.8.1:
 
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 0eee8d758..3afa06077 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -1,15 +1,26 @@
 import logging
 from collections import defaultdict
 
+from tldextract import TLDExtract
+
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
+from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 
 
 logger = logging.getLogger(__name__)
 
 
+_split_domain = TLDExtract(include_psl_private_domains=True)
+
+
+def _is_public_domain(domain):
+    parts = _split_domain(domain)
+    return not parts.domain
+
+
 class CookiesMiddleware:
     """This middleware enables working with sites that need cookies"""
 
@@ -23,14 +34,29 @@ class CookiesMiddleware:
             raise NotConfigured
         return cls(crawler.settings.getbool('COOKIES_DEBUG'))
 
+    def _process_cookies(self, cookies, *, jar, request):
+        for cookie in cookies:
+            cookie_domain = cookie.domain
+            if cookie_domain.startswith('.'):
+                cookie_domain = cookie_domain[1:]
+
+            request_domain = urlparse_cached(request).hostname.lower()
+
+            if cookie_domain and _is_public_domain(cookie_domain):
+                if cookie_domain != request_domain:
+                    continue
+                cookie.domain = request_domain
+
+            jar.set_cookie_if_ok(cookie, request)
+
     def process_request(self, request, spider):
         if request.meta.get('dont_merge_cookies', False):
             return
 
         cookiejarkey = request.meta.get("cookiejar")
         jar = self.jars[cookiejarkey]
-        for cookie in self._get_request_cookies(jar, request):
-            jar.set_cookie_if_ok(cookie, request)
+        cookies = self._get_request_cookies(jar, request)
+        self._process_cookies(cookies, jar=jar, request=request)
 
         # set Cookie header
         request.headers.pop('Cookie', None)
@@ -44,7 +70,9 @@ class CookiesMiddleware:
         # extract cookies from Set-Cookie and drop invalid/expired cookies
         cookiejarkey = request.meta.get("cookiejar")
         jar = self.jars[cookiejarkey]
-        jar.extract_cookies(response, request)
+        cookies = jar.make_cookies(response, request)
+        self._process_cookies(cookies, jar=jar, request=request)
+
         self._debug_set_cookie(response, spider)
 
         return response
diff --git a/setup.py b/setup.py
index 3a6ff2836..d86c0f285 100644
--- a/setup.py
+++ b/setup.py
@@ -32,6 +32,7 @@ install_requires = [
     'protego>=0.1.15',
     'itemadapter>=0.1.0',
     'setuptools',
+    'tldextract',
 ]
 extras_require = {}
 cpython_dependencies = [
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 1747f3b94..ba7453255 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -15,6 +15,48 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
 
 
+def _cookie_to_set_cookie_value(cookie):
+    """Given a cookie defined as a dictionary with name and value keys, and
+    optional path and domain keys, return the equivalent string that can be
+    associated to a ``Set-Cookie`` header."""
+    decoded = {}
+    for key in ("name", "value", "path", "domain"):
+        if cookie.get(key) is None:
+            if key in ("name", "value"):
+                return
+            continue
+        if isinstance(cookie[key], (bool, float, int, str)):
+            decoded[key] = str(cookie[key])
+        else:
+            try:
+                decoded[key] = cookie[key].decode("utf8")
+            except UnicodeDecodeError:
+                decoded[key] = cookie[key].decode("latin1", errors="replace")
+
+    cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}"
+    for key, value in decoded.items():  # path, domain
+        cookie_str += f"; {key.capitalize()}={value}"
+    return cookie_str
+
+
+def _cookies_to_set_cookie_list(cookies):
+    """Given a group of cookie defined either as a dictionary or as a list of
+    dictionaries (i.e. in a format supported by the cookies parameter of
+    Request), return the equivalen list of strings that can be associated to a
+    ``Set-Cookie`` header."""
+    if not cookies:
+        return []
+    if isinstance(cookies, dict):
+        cookies = ({"name": k, "value": v} for k, v in cookies.items())
+    return filter(
+        None,
+        (
+            _cookie_to_set_cookie_value(cookie)
+            for cookie in cookies
+        )
+    )
+
+
 class CookiesMiddlewareTest(TestCase):
 
     def assertCookieValEqual(self, first, second, msg=None):
@@ -523,3 +565,131 @@ class CookiesMiddlewareTest(TestCase):
             {'url': 'https://example.com', 'status': 302},
             cookies2=False,
         )
+
+    def _test_user_set_cookie_domain_followup(
+        self,
+        url1,
+        url2,
+        domain,
+        *,
+        cookies1,
+        cookies2,
+    ):
+        input_cookies = [
+            {
+                'name': 'a',
+                'value': 'b',
+                'domain': domain,
+            }
+        ]
+
+        request1 = Request(url1, cookies=input_cookies)
+        self.mw.process_request(request1, self.spider)
+        cookies = request1.headers.get('Cookie')
+        self.assertEqual(cookies, b"a=b" if cookies1 else None)
+
+        request2 = Request(url2)
+        self.mw.process_request(request2, self.spider)
+        cookies = request2.headers.get('Cookie')
+        self.assertEqual(cookies, b"a=b" if cookies2 else None)
+
+    def test_user_set_cookie_domain_suffix_private(self):
+        self._test_user_set_cookie_domain_followup(
+            'https://books.toscrape.com',
+            'https://quotes.toscrape.com',
+            'toscrape.com',
+            cookies1=True,
+            cookies2=True,
+        )
+
+    def test_user_set_cookie_domain_suffix_public_period(self):
+        self._test_user_set_cookie_domain_followup(
+            'https://foo.co.uk',
+            'https://bar.co.uk',
+            'co.uk',
+            cookies1=False,
+            cookies2=False,
+        )
+
+    def test_user_set_cookie_domain_suffix_public_private(self):
+        self._test_user_set_cookie_domain_followup(
+            'https://foo.blogspot.com',
+            'https://bar.blogspot.com',
+            'blogspot.com',
+            cookies1=False,
+            cookies2=False,
+        )
+
+    def test_user_set_cookie_domain_public_period(self):
+        self._test_user_set_cookie_domain_followup(
+            'https://co.uk',
+            'https://co.uk',
+            'co.uk',
+            cookies1=True,
+            cookies2=True,
+        )
+
+    def _test_server_set_cookie_domain_followup(
+        self,
+        url1,
+        url2,
+        domain,
+        *,
+        cookies,
+    ):
+        request1 = Request(url1)
+        self.mw.process_request(request1, self.spider)
+
+        input_cookies = [
+            {
+                'name': 'a',
+                'value': 'b',
+                'domain': domain,
+            }
+        ]
+
+        headers = {
+            'Set-Cookie': _cookies_to_set_cookie_list(input_cookies),
+        }
+        response = Response(url1, status=200, headers=headers)
+        self.assertEqual(
+            self.mw.process_response(request1, response, self.spider),
+            response,
+        )
+
+        request2 = Request(url2)
+        self.mw.process_request(request2, self.spider)
+        actual_cookies = request2.headers.get('Cookie')
+        self.assertEqual(actual_cookies, b"a=b" if cookies else None)
+
+    def test_server_set_cookie_domain_suffix_private(self):
+        self._test_server_set_cookie_domain_followup(
+            'https://books.toscrape.com',
+            'https://quotes.toscrape.com',
+            'toscrape.com',
+            cookies=True,
+        )
+
+    def test_server_set_cookie_domain_suffix_public_period(self):
+        self._test_server_set_cookie_domain_followup(
+            'https://foo.co.uk',
+            'https://bar.co.uk',
+            'co.uk',
+            cookies=False,
+        )
+
+    def test_server_set_cookie_domain_suffix_public_private(self):
+        self._test_server_set_cookie_domain_followup(
+            'https://foo.blogspot.com',
+            'https://bar.blogspot.com',
+            'blogspot.com',
+            cookies=False,
+        )
+
+    def test_server_set_cookie_domain_public_period(self):
+        self._test_server_set_cookie_domain_followup(
+            'https://co.uk',
+            'https://co.uk',
+            'co.uk',
+            cookies=True,
+        )

From 6b63e7c14758fdc59f37cb6c2c9b88abebe8606f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 1 Mar 2022 12:43:11 +0100
Subject: [PATCH 551/716] =?UTF-8?q?Bump=20version:=202.5.0=20=E2=86=92=202?=
 =?UTF-8?q?.6.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index d9e4a2831..5a5b51a01 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.5.0
+current_version = 2.6.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 437459cd9..e70b4523a 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.5.0
+2.6.0

From 84853c4fa6eb30bdcba0f70b4426994d731509fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 1 Mar 2022 13:01:20 +0100
Subject: [PATCH 552/716] bandit: allow-list B324 for the time being

---
 .bandit.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.bandit.yml b/.bandit.yml
index 243379b0b..41f1bb597 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -8,6 +8,7 @@ skips:
 - B311
 - B320
 - B321
+- B324
 - B402  # https://github.com/scrapy/scrapy/issues/4180
 - B403
 - B404

From d60636d0de94c5a08c25d1d6820faed0b45506b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 1 Mar 2022 13:06:58 +0100
Subject: [PATCH 553/716] Fix redirect handling regression

---
 scrapy/downloadermiddlewares/redirect.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index fcd6c298b..c8c84ffb2 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -12,11 +12,10 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
 logger = logging.getLogger(__name__)
 
 
-def _build_redirect_request(source_request, *, url, method=None, body=None):
+def _build_redirect_request(source_request, *, url, **kwargs):
     redirect_request = source_request.replace(
         url=url,
-        method=method,
-        body=body,
+        **kwargs,
         cookies=None,
     )
     if 'Cookie' in redirect_request.headers:

From fab3e907297abd89106fb040c1c0c6a24b9522a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 1 Mar 2022 13:41:20 +0100
Subject: [PATCH 554/716] Cover 2.6.1 in the release notes

---
 docs/news.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 9590fb1c4..5d92067b5 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,15 @@
 Release notes
 =============
 
+.. _release-2.6.1:
+
+Scrapy 2.6.1 (2022-03-01)
+-------------------------
+
+Fixes a regression introduced in 2.6.0 that would unset the request method when
+following redirects.
+
+
 .. _release-2.6.0:
 
 Scrapy 2.6.0 (2022-03-01)

From 23537a0f9580bfb28ac5d8b88f37df47e838f463 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 1 Mar 2022 13:48:40 +0100
Subject: [PATCH 555/716] =?UTF-8?q?Bump=20version:=202.6.0=20=E2=86=92=202?=
 =?UTF-8?q?.6.1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 5a5b51a01..1d9b9c02f 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.6.0
+current_version = 2.6.1
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index e70b4523a..6a6a3d8e3 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.6.0
+2.6.1

From 50c8becbe02e6e71a6e7a57af0b28bcf38d9a3c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 1 Mar 2022 17:29:08 +0100
Subject: [PATCH 556/716] Freeze and upgrade CI packages (#5429)

---
 tests/requirements.txt | 2 +-
 tox.ini                | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/requirements.txt b/tests/requirements.txt
index bd72c8c46..398d1d16d 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -3,7 +3,7 @@ attrs
 dataclasses; python_version == '3.6'
 pyftpdlib
 pytest
-pytest-cov
+pytest-cov==3.0.0
 pytest-xdist
 sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures
diff --git a/tox.ini b/tox.ini
index fcd3563b2..db151f215 100644
--- a/tox.ini
+++ b/tox.ini
@@ -49,7 +49,7 @@ commands =
 [testenv:security]
 basepython = python3
 deps =
-    bandit
+    bandit==1.7.3
 commands =
     bandit -r -c .bandit.yml {posargs:scrapy}
 
@@ -68,7 +68,7 @@ commands =
 basepython = python3
 deps =
     {[testenv:extra-deps]deps}
-    pylint==2.12.1
+    pylint==2.12.2
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 

From ccdbb795ff2fef0ff89c1fa478ad2d7c52ef64be Mon Sep 17 00:00:00 2001
From: Florentin <arsene.florentin.ichb@gmail.com>
Date: Tue, 1 Mar 2022 22:01:55 +0100
Subject: [PATCH 557/716] Recommend Common Crawl instead of Google Cache

---
 docs/topics/practices.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 1a9d56143..d0207fd18 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -262,7 +262,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
 * disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use
   cookies to spot bot behaviour
 * use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting.
-* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites
+* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
   directly
 * use a pool of rotating IPs. For example, the free `Tor project`_ or paid
   services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
@@ -277,7 +277,7 @@ If you are still unable to prevent your bot getting banned, consider contacting
 .. _Tor project: https://www.torproject.org/
 .. _commercial support: https://scrapy.org/support/
 .. _ProxyMesh: https://proxymesh.com/
-.. _Google cache: http://www.googleguide.com/cached_pages.html
+.. _Common Crawl: https://commoncrawl.org/
 .. _testspiders: https://github.com/scrapinghub/testspiders
 .. _scrapoxy: https://scrapoxy.io/
 .. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/

From 3ecbea4b876ed084f30bb4063e1993dd9c3cdb8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 2 Mar 2022 16:06:49 +0100
Subject: [PATCH 558/716] CrawlerProcess: initiate the reactor only once

---
 scrapy/crawler.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index a638254f1..9939a19eb 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -290,6 +290,7 @@ class CrawlerProcess(CrawlerRunner):
         super().__init__(settings)
         configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)
+        self._initiated_reactor = False
 
     def _signal_shutdown(self, signum, _):
         from twisted.internet import reactor
@@ -310,7 +311,9 @@ class CrawlerProcess(CrawlerRunner):
     def _create_crawler(self, spidercls):
         if isinstance(spidercls, str):
             spidercls = self.spider_loader.load(spidercls)
-        return Crawler(spidercls, self.settings, init_reactor=True)
+        init_reactor = not self._initiated_reactor
+        self._initiated_reactor = True
+        return Crawler(spidercls, self.settings, init_reactor=init_reactor)
 
     def start(self, stop_after_crawl=True, install_signal_handlers=True):
         """

From 96fc4dae15181695c58040389fa502857a2b0df8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 2 Mar 2022 16:14:39 +0100
Subject: [PATCH 559/716] CrawlerProcess: test a multi-spider scenario

---
 tests/CrawlerProcess/multi.py | 16 ++++++++++++++++
 tests/test_crawler.py         |  6 ++++++
 2 files changed, 22 insertions(+)
 create mode 100644 tests/CrawlerProcess/multi.py

diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py
new file mode 100644
index 000000000..aaa1af5c5
--- /dev/null
+++ b/tests/CrawlerProcess/multi.py
@@ -0,0 +1,16 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8f6227109..957525382 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -302,6 +302,12 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn('Spider closed (finished)', log)
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    def test_multi(self):
+        log = self.run_script('multi.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertNotIn("ReactorAlreadyInstalledError", log)
+
     def test_asyncio_enabled_no_reactor(self):
         log = self.run_script('asyncio_enabled_no_reactor.py')
         self.assertIn('Spider closed (finished)', log)

From 3bf6baeaa705ddc2d3417206f3db816dabadf5a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 2 Mar 2022 17:03:41 +0100
Subject: [PATCH 560/716] =?UTF-8?q?initiated=20=E2=86=92=20initialized?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/crawler.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 9939a19eb..d669d93a8 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -290,7 +290,7 @@ class CrawlerProcess(CrawlerRunner):
         super().__init__(settings)
         configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)
-        self._initiated_reactor = False
+        self._initialized_reactor = False
 
     def _signal_shutdown(self, signum, _):
         from twisted.internet import reactor
@@ -311,8 +311,8 @@ class CrawlerProcess(CrawlerRunner):
     def _create_crawler(self, spidercls):
         if isinstance(spidercls, str):
             spidercls = self.spider_loader.load(spidercls)
-        init_reactor = not self._initiated_reactor
-        self._initiated_reactor = True
+        init_reactor = not self._initialized_reactor
+        self._initialized_reactor = True
         return Crawler(spidercls, self.settings, init_reactor=init_reactor)
 
     def start(self, stop_after_crawl=True, install_signal_handlers=True):

From 62a00812669aa0906aa0d4e9a4c2e87be3b74975 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 7 Mar 2022 12:00:44 +0100
Subject: [PATCH 561/716] engine: prevent slot method call after unsetting the
 slot

---
 scrapy/core/engine.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index f9de7ee23..6602f661d 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -136,7 +136,9 @@ class ExecutionEngine:
         self.paused = False
 
     def _next_request(self) -> None:
-        assert self.slot is not None  # typing
+        if self.slot is None:
+            return
+
         assert self.spider is not None  # typing
 
         if self.paused:
@@ -184,7 +186,8 @@ class ExecutionEngine:
         d.addErrback(lambda f: logger.info('Error while removing request from slot',
                                            exc_info=failure_to_exc_info(f),
                                            extra={'spider': self.spider}))
-        d.addBoth(lambda _: self.slot.nextcall.schedule())
+        slot = self.slot
+        d.addBoth(lambda _: slot.nextcall.schedule())
         d.addErrback(lambda f: logger.info('Error while scheduling new request',
                                            exc_info=failure_to_exc_info(f),
                                            extra={'spider': self.spider}))

From d469214fe73574d35521e8e87963629a6c12bcd8 Mon Sep 17 00:00:00 2001
From: Ali Rastegar <68519335+VolVox99@users.noreply.github.com>
Date: Tue, 8 Mar 2022 01:29:22 -0800
Subject: [PATCH 562/716] Update tutorial.rst (#5442)

Fixed typo
---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 5697b9608..cde1b1ef4 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -488,7 +488,7 @@ The `JSON Lines`_ format is useful because it's stream-like, you can easily
 append new records to it. It doesn't have the same problem of JSON when you run
 twice. Also, as each record is a separate line, you can process big files
 without having to fit everything in memory, there are tools like `JQ`_ to help
-doing that at the command-line.
+do that at the command-line.
 
 In small projects (like the one in this tutorial), that should be enough.
 However, if you want to perform more complex things with the scraped items, you

From e264cc30d1e73d53de2e4048d7d9b6cffd59f8f0 Mon Sep 17 00:00:00 2001
From: NaincyKumariKnoldus
 <87004609+NaincyKumariKnoldus@users.noreply.github.com>
Date: Thu, 10 Mar 2022 19:24:33 +0530
Subject: [PATCH 563/716] removed  the pywin32 docs section (#5370)

---
 docs/faq.rst | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 8283cab11..8a9ba809b 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -94,15 +94,6 @@ How can I scrape an item with attributes in different pages?
 
 See :ref:`topics-request-response-ref-request-callback-arguments`.
 
-
-Scrapy crashes with: ImportError: No module named win32api
-----------------------------------------------------------
-
-You need to install `pywin32`_ because of `this Twisted bug`_.
-
-.. _pywin32: https://sourceforge.net/projects/pywin32/
-.. _this Twisted bug: https://twistedmatrix.com/trac/ticket/3707
-
 How can I simulate a user login in my spider?
 ---------------------------------------------
 

From c1d4be8cb5ed610fef4a2af60f9b654faab7a243 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Tue, 15 Mar 2022 07:30:30 -0300
Subject: [PATCH 564/716] =?UTF-8?q?Restore=20=E2=80=98-o=20-=E2=80=99=20su?=
 =?UTF-8?q?pport=20(#5445)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/cmdline.py      | 17 +++++++++++++----
 tests/test_commands.py | 15 +++++++++++++++
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 491c4beab..5ee1f0f44 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -14,6 +14,15 @@ from scrapy.utils.project import inside_project, get_project_settings
 from scrapy.utils.python import garbage_collect
 
 
+class ScrapyArgumentParser(argparse.ArgumentParser):
+    def _parse_optional(self, arg_string):
+        # if starts with -: it means that is a parameter not a argument
+        if arg_string[:2] == '-:':
+            return None
+
+        return super()._parse_optional(arg_string)
+
+
 def _iter_command_classes(module_name):
     # TODO: add `name` attribute to commands and and merge this function with
     # scrapy.utils.spider.iter_spider_classes
@@ -131,10 +140,10 @@ def execute(argv=None, settings=None):
         sys.exit(2)
 
     cmd = cmds[cmdname]
-    parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
-                                     usage=f"scrapy {cmdname} {cmd.syntax()}",
-                                     conflict_handler='resolve',
-                                     description=cmd.long_desc())
+    parser = ScrapyArgumentParser(formatter_class=ScrapyHelpFormatter,
+                                  usage=f"scrapy {cmdname} {cmd.syntax()}",
+                                  conflict_handler='resolve',
+                                  description=cmd.long_desc())
     settings.setdict(cmd.default_settings, priority='command')
     cmd.settings = settings
     cmd.add_options(parser)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 7cd19b29a..b5e6c2b8b 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -770,6 +770,21 @@ class MySpider(scrapy.Spider):
         log = self.get_log(spider_code, args=args)
         self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
 
+    def test_output_stdout(self):
+        spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+
+    def start_requests(self):
+        self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
+        return []
+"""
+        args = ['-o', '-:json']
+        log = self.get_log(spider_code, args=args)
+        self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log)
+
 
 @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
 class WindowsRunSpiderCommandTest(RunSpiderCommandTest):

From b59a69be1790f138afe89df3dfed17ac48384d8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 15 Mar 2022 12:10:41 +0100
Subject: [PATCH 565/716] Test that a low CLOSEPIDER_TIMEOUT does not raise an
 exception

---
 tests/test_engine.py | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/tests/test_engine.py b/tests/test_engine.py
index fa7d0c8d4..b8fd341f6 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -12,9 +12,11 @@ module with the ``runserver`` argument::
 
 import os
 import re
+import subprocess
 import sys
 import warnings
 from collections import defaultdict
+from threading import Timer
 from urllib.parse import urlparse
 
 import attr
@@ -502,6 +504,37 @@ class EngineTest(unittest.TestCase):
             self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
             self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated")
 
+    def test_short_timeout(self):
+        args = (
+            sys.executable,
+            '-m',
+            'scrapy.cmdline',
+            'fetch',
+            '-s',
+            'CLOSESPIDER_TIMEOUT=0.001',
+            '-s',
+            'LOG_LEVEL=DEBUG',
+            'http://toscrape.com',
+        )
+        p = subprocess.Popen(
+            args,
+            stderr=subprocess.PIPE,
+        )
+
+        def kill_proc():
+            p.kill()
+            p.communicate()
+            assert False, 'Command took too much time to complete'
+
+        timer = Timer(15, kill_proc)
+        try:
+            timer.start()
+            _, stderr = p.communicate()
+        finally:
+            timer.cancel()
+
+        self.assertNotIn(b'Traceback', stderr)
+
 
 if __name__ == "__main__":
     if len(sys.argv) > 1 and sys.argv[1] == 'runserver':

From 78ba4b033b016be7fbb22bfa9e6d5d389380e6d4 Mon Sep 17 00:00:00 2001
From: Yann Defretin <yann@defret.in>
Date: Wed, 16 Mar 2022 15:14:24 +0100
Subject: [PATCH 566/716] fixed detection of extension like ".tar.gz" in URL

---
 scrapy/utils/url.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index a6a2a9e8b..bae5a9433 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -5,7 +5,6 @@ library.
 Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
-import posixpath
 import re
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 
@@ -31,8 +30,8 @@ def url_is_from_spider(url, spider):
 
 
 def url_has_any_extension(url, extensions):
-    return posixpath.splitext(parse_url(url).path)[1].lower() in extensions
-
+    """Return True if the url ends with one of the extensions provided"""
+    return any(parse_url(url).path.lower().endswith(ext) for ext in extensions)
 
 def parse_url(url, encoding=None):
     """Return urlparsed url from the given argument (which could be an already

From fd08bb6cd99f16c9ce433583d4698801dc7e0ac3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 16 Mar 2022 14:34:57 +0100
Subject: [PATCH 567/716] Refactor the asynchronous process_spider_output
 documentation

---
 docs/topics/coroutines.rst        | 157 +++++++++++++++++-------------
 docs/topics/spider-middleware.rst |  22 +++--
 2 files changed, 102 insertions(+), 77 deletions(-)

diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 073b6bd9a..361cd5e60 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -17,9 +17,12 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :class:`~scrapy.Request` callbacks.
 
+    If you are using any custom or third-party :ref:`spider middleware
+    <topics-spider-middleware>`, see :ref:`sync-async-spider-middleware`.
+
     .. versionchanged:: VERSION
-       Output of async callbacks is now processed asynchronously instead of collecting
-       all of it first.
+       Output of async callbacks is now processed asynchronously instead of
+       collecting all of it first.
 
 -   The :meth:`process_item` method of
     :ref:`item pipelines <topics-item-pipeline>`.
@@ -34,18 +37,26 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :ref:`Signal handlers that support deferreds <signal-deferred>`.
 
--   The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
-    method of :ref:`spider middlewares <custom-spider-middleware>`. See
-    :ref:`async-spider-middlewares`.
+-   The
+    :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+    method of :ref:`spider middlewares <topics-spider-middleware>`.
+
+    It must be defined as an :term:`asynchronous generator`. The input
+    ``result`` parameter is an :term:`asynchronous iterable`.
+
+    See also :ref:`sync-async-spider-middleware` and
+    :ref:`universal-spider-middleware`.
 
     .. versionadded:: VERSION
 
-Usage
-=====
+General usage
+=============
 
-There are several use cases for coroutines in Scrapy. Code that would
-return Deferreds when written for previous Scrapy versions, such as downloader
-middlewares and signal handlers, can be rewritten to be shorter and cleaner::
+There are several use cases for coroutines in Scrapy.
+
+Code that would return Deferreds when written for previous Scrapy versions,
+such as downloader middlewares and signal handlers, can be rewritten to be
+shorter and cleaner::
 
     from itemadapter import ItemAdapter
 
@@ -110,46 +121,73 @@ Common use cases for asynchronous code include:
 
 .. _aio-libs: https://github.com/aio-libs
 
-.. _async-spider-middlewares:
 
-Asynchronous spider middlewares
-===============================
+.. _sync-async-spider-middleware:
+
+Mixing synchronous and asynchronous spider middlewares
+======================================================
 
 .. versionadded:: VERSION
-.. note:: This currently applies to
-          :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`.
-          In the future it will also apply to
-          :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests`.
 
-Middleware methods discussed here can take and return async iterables. They can
-return the same type of iterable or they can take a normal one and return an
-async one. If such method needs to return an async iterable it must be an async
-generator, not just a coroutine that returns an iterable.
+The output of a :class:`~scrapy.Request` callback is passed as the ``result``
+parameter to the
+:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
+of the first :ref:`spider middleware <topics-spider-middleware>` from the
+:ref:`list of active spider middlewares <topics-spider-middleware-setting>`.
+Then the output of that ``process_spider_output`` method is passed to the
+``process_spider_output`` method of the next spider middleware, and so on for
+every active spider middleware.
 
-As the result of a middleware method is passed to the same method of the next
-middleware, it needs to be adapted if the second method expects a different
-type. Scrapy will do this transparently:
+Scrapy supports mixing :ref:`coroutine methods <async>` and synchronous methods
+in this chain of calls.
 
-* A normal iterable is wrapped into an async one which shouldn't cause any side
-  effects.
-* An async iterable is downgraded to a normal one by waiting until all results
-  are available and wrapping them in a normal iterable. This is problematic
-  because it pauses the normal middleware processing for this iterable and
-  because all results can be skipped if exceptions are raised during
-  processing. This case emits a warning and will be deprecated and then removed
-  in a later Scrapy version.
-* Async iterables returned from
-  :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception`
-  won't be downgraded, an exception will be raised if that is needed.
+However, if any of the ``process_spider_output`` methods is defined as a
+synchronous method, and the previous ``Request`` callback or
+``process_spider_output`` method is a coroutine, there are some drawbacks to
+the asynchronous-to-synchronous conversion that Scrapy does so that the
+synchronous ``process_spider_output`` method gets a synchronous iterable as its
+``result`` parameter:
 
-As downgrading is undesirable, here is the proposed way to avoid it. If all
-middlewares, including 3rd-party ones, support async iterables as input, no
-downgrading will happen. But removing normal iterable support (making the
-method a coroutine) from a middleware published as a separate project or used
-internally in projects for older Scrapy versions breaks backwards
-compatibility. So, as an interim measure (it will be deprecated and then
-removed in a later Scrapy version), a middleware can provide both sync and
-async methods in the following form::
+-   The whole output of the previous ``Request`` callback or
+    ``process_spider_output`` method is awaited at this point.
+
+-   If an exception raises while awaiting the output of the previous
+    ``Request`` callback or ``process_spider_output`` method, none of that
+    output will be processed.
+
+Asynchronous-to-synchronous conversions are supported for backward
+compatibility, but they are deprecated and will stop working in a future
+version of Scrapy.
+
+To avoid asynchronous-to-synchronous conversion, when defining ``Request``
+callbacks as coroutine methods or when using spider middlewares whose
+``process_spider_output`` method is an :term:`asynchronous generator`, all
+active spider middlewares must either have their ``process_spider_output``
+method defined as an asynchronous generator or :ref:`define a
+process_spider_output_async method <universal-spider-middleware>`.
+
+.. note:: When using third-party spider middlewares that only define a
+          synchronous ``process_spider_output`` method, consider
+          :ref:`making them universal <universal-spider-middleware>` through
+          :ref:`subclassing <tut-inheritance>`.
+
+
+.. _universal-spider-middleware:
+
+Universal spider middleware
+===========================
+
+.. versionadded:: VERSION
+
+To allow writing a spider middleware that supports asynchronous execution of
+its ``process_spider_output`` method in Scrapy VERSION and later (avoiding
+:ref:`asynchronous-to-synchronous conversions <sync-async-spider-middleware>`)
+while maintaining support for older Scrapy versions, you may define
+``process_spider_output`` as a synchronous method and define an
+:term:`asynchronous generator` version of that method with an alternative name:
+``process_spider_output_async``.
+
+For example::
 
     class UniversalSpiderMiddleware:
         def process_spider_output(self, response, result, spider):
@@ -162,28 +200,13 @@ async methods in the following form::
                 # ... do something with r
                 yield r
 
-In this case normal and async iterables will be passed to the respective
-methods without any wrapping or downgrading, and in older versions of Scrapy
-the coroutine method will just be ignored. When the backwards compatibility is
-no longer needed the non-coroutine method can be dropped and the coroutine one
-renamed to the normal name. It may be possible to extract common code from both
-methods to reduce code duplication, as in the simplest case the only difference
-between them will be ``for`` vs ``async for``.
+.. note:: This is an interim measure to allow, for a time, to write code that
+          works in Scrapy VERSION and later without requiring
+          asynchronous-to-synchronous conversions, and works in earlier Scrapy
+          versions as well.
 
-So, to recap:
-
-* If you don't intend to use async callbacks or middlewares containing async
-  code in your project, nothing should change for you yet. At some point in the
-  future some of the 3rd-party middlewares you use may drop backwards
-  compatibility, which shouldn't lead to immediate problems but may be a sign
-  to start converting your code to ``async def`` too.
-* If you maintain a middleware that can be used with projects you can't control
-  (e.g. one you published for other people to use, or one that needs to support
-  some old project that can't be modernized), we recommend adding a
-  ``process_spider_output_async`` method so that the amount of unnecessary
-  iterable conversions is reduced but no compatibility is broken.
-* If you use async callbacks, try to make sure all middlewares support them.
-  Note that you can modernize 3rd-party middlewares by subclassing them.
-* If you want to write and publish a middleware that requires async code, you
-  should write in the docs that the minimum support Scrapy version is VERSION
-  (maybe even check this at the run time, using :attr:`scrapy.__version__`).
+          In some future version of Scrapy, however, this feature will be
+          deprecated and, eventually, in a later version of Scrapy, this
+          feature will be removed, and all spider middlewares will be expected
+          to define their ``process_spider_output`` method as an asynchronous
+          generator.
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index edfc2e4bb..787545ed2 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -98,10 +98,6 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
     .. method:: process_spider_output(response, result, spider)
 
-        .. versionchanged:: VERSION
-           Since VERSION this can take and return an :term:`python:asynchronous
-           iterable`.
-
         This method is called with the results returned from the Spider, after
         it has processed the response.
 
@@ -109,8 +105,15 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         :class:`~scrapy.Request` objects and :ref:`item objects
         <topics-items>`.
 
-        .. note:: When defined as a :ref:`coroutine <async>`, this method needs
-                  to be an async generator, not just return an iterable.
+        .. versionchanged:: VERSION
+           This method may be defined as an :term:`asynchronous generator`, in
+           which case ``result`` is an :term:`asynchronous iterable`.
+
+        Consider defining this method as an :term:`asynchronous generator`,
+        which will be a requirement in a future version of Scrapy. However, if
+        you wish your spider middleware to work with Scrapy versions earlier
+        than Scrapy VERSION, :ref:`make your spider middleware universal
+        <universal-spider-middleware>` instead.
 
         :param response: the response which generated this output from the
           spider
@@ -127,10 +130,9 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
         .. versionadded:: VERSION
 
-        If exists, this methid will be called instead of
-        :meth:`process_spider_output` when ``result`` is an async iterable.
-        If this method exists, it must be a coroutine while
-        :meth:`process_spider_output` must not be a coroutine.
+        If defined, this method must be an :term:`asynchronous generator`,
+        which will be called instead of :meth:`process_spider_output` if
+        ``result`` is an :term:`asynchronous iterable`.
 
     .. method:: process_spider_exception(response, exception, spider)
 

From c961438d5d9998344460d930ea502fae40553043 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 16 Mar 2022 18:45:56 +0100
Subject: [PATCH 568/716] tests: cover scenarios of bad results from
 process_spider_output

---
 scrapy/core/spidermw.py        | 19 ++++++++----
 tests/test_spidermiddleware.py | 54 +++++++++++++++++++++++++++-------
 2 files changed, 58 insertions(+), 15 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 6075670b0..1aa02f29f 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -4,7 +4,7 @@ Spider Middleware manager
 See documentation in docs/topics/spider-middleware.rst
 """
 import logging
-from inspect import isasyncgenfunction
+from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice
 from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast
 
@@ -61,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             try:
                 result = method(response=response, spider=spider)
                 if result is not None:
-                    msg = (f"Middleware {method.__qualname__} must return None "
+                    msg = (f"{method.__qualname__} must return None "
                            f"or raise an exception, got {type(result)}")
                     raise _InvalidOutput(msg)
             except _InvalidOutput:
@@ -129,7 +129,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             elif result is None:
                 continue
             else:
-                msg = (f"Middleware {method.__qualname__} must return None "
+                msg = (f"{method.__qualname__} must return None "
                        f"or an iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
         return _failure
@@ -197,8 +197,17 @@ class SpiderMiddlewareManager(MiddlewareManager):
             if _isiterable(result):
                 result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
             else:
-                msg = (f"Middleware {method.__qualname__} must return an "
-                       f"iterable, got {type(result)}")
+                if iscoroutine(result):
+                    result.close()  # Silence warning about not awaiting
+                    msg = (
+                        f"{method.__qualname__} must be an asynchronous "
+                        f"generator (i.e. use yield)"
+                    )
+                else:
+                    msg = (
+                        f"{method.__qualname__} must return an iterable, got "
+                        f"{type(result)}"
+                    )
                 raise _InvalidOutput(msg)
             last_result_is_async = isinstance(result, AsyncIterable)
 
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index f9f2b6642..ed0912b82 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -123,6 +123,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
             start_index = 10
         return {i: c for c, i in enumerate(mw_classes, start=start_index)}
 
+    def _scrape_func(self, *args, **kwargs):
+        yield {'foo': 1}
+        yield {'foo': 2}
+        yield {'foo': 3}
+
     @defer.inlineCallbacks
     def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
         setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
@@ -201,11 +206,6 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
     MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
     MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
 
-    def _scrape_func(self, *args, **kwargs):
-        yield {'foo': 1}
-        yield {'foo': 2}
-        yield {'foo': 3}
-
     def test_simple(self):
         """ Simple mw """
         return self._test_simple_base(self.MW_SIMPLE)
@@ -285,6 +285,45 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
                                       downgrade=True)
 
 
+class ProcessSpiderOutputNonIterableMiddleware:
+    def process_spider_output(self, response, result, spider):
+        return
+
+
+class ProcessSpiderOutputCoroutineMiddleware:
+    async def process_spider_output(self, response, result, spider):
+        results = []
+        for r in result:
+            results.append(r)
+        return results
+
+
+class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
+
+    @defer.inlineCallbacks
+    def test_non_iterable(self):
+        with self.assertRaisesRegex(
+            _InvalidOutput,
+            (
+                "\.process_spider_output must return an iterable, got <class "
+                "'NoneType'>"
+            ),
+        ):
+            yield self._get_middleware_result(
+                ProcessSpiderOutputNonIterableMiddleware,
+            )
+
+    @defer.inlineCallbacks
+    def test_coroutine(self):
+        with self.assertRaisesRegex(
+            _InvalidOutput,
+            "\.process_spider_output must be an asynchronous generator",
+        ):
+            yield self._get_middleware_result(
+                ProcessSpiderOutputCoroutineMiddleware,
+            )
+
+
 class ProcessStartRequestsSimpleMiddleware:
     def process_start_requests(self, start_requests, spider):
         for r in start_requests:
@@ -387,11 +426,6 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
     MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
     MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
 
-    def _scrape_func(self, *args, **kwargs):
-        yield {'foo': 1}
-        yield {'foo': 2}
-        yield {'foo': 3}
-
     @defer.inlineCallbacks
     def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
         setting = self._construct_mw_setting(*mw_classes, start_index=start_index)

From b78e6915c6b259b31d3c37cae1529e85e797c964 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 16 Mar 2022 20:17:25 +0100
Subject: [PATCH 569/716] Clarify that without async-to-sync conversions items
 yielded before an exception are processed

---
 docs/topics/coroutines.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 361cd5e60..efc4566a0 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -155,6 +155,9 @@ synchronous ``process_spider_output`` method gets a synchronous iterable as its
     ``Request`` callback or ``process_spider_output`` method, none of that
     output will be processed.
 
+    This contrasts with the regular behavior, where all items yielded before
+    an exception raises are processed.
+
 Asynchronous-to-synchronous conversions are supported for backward
 compatibility, but they are deprecated and will stop working in a future
 version of Scrapy.

From 5b4b8b6fb12874d4a0a11c261341639c9af95b10 Mon Sep 17 00:00:00 2001
From: Yann Defretin <yann@defret.in>
Date: Wed, 16 Mar 2022 22:32:05 +0100
Subject: [PATCH 570/716] added test for new url_has_any_extension function

---
 tests/test_utils_url.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 144c7bd76..58e2be622 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,6 +1,8 @@
 import unittest
 
+from scrapy.linkextractors import IGNORED_EXTENSIONS
 from scrapy.spiders import Spider
+from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.url import (
     add_http_if_no_scheme,
     guess_scheme,
@@ -8,9 +10,9 @@ from scrapy.utils.url import (
     strip_url,
     url_is_from_any_domain,
     url_is_from_spider,
+    url_has_any_extension,
 )
 
-
 __doctests__ = ['scrapy.utils.url']
 
 
@@ -81,6 +83,15 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider))
         self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider))
 
+    def test_url_has_any_extension(self):
+        deny_extensions = {'.' + e for e in arg_to_iter(IGNORED_EXTENSIONS)}
+        self.assertTrue(url_has_any_extension("http://www.example.com/archive.tar.gz", deny_extensions))
+        self.assertTrue(url_has_any_extension("http://www.example.com/page.doc", deny_extensions))
+        self.assertTrue(url_has_any_extension("http://www.example.com/page.pdf", deny_extensions))
+        self.assertFalse(url_has_any_extension("http://www.example.com/page.htm", deny_extensions))
+        self.assertFalse(url_has_any_extension("http://www.example.com/", deny_extensions))
+        self.assertFalse(url_has_any_extension("http://www.example.com/page.doc.html", deny_extensions))
+
 
 class AddHttpIfNoScheme(unittest.TestCase):
 

From 9a28eb0bad1acf986d997905a410058f77911b7c Mon Sep 17 00:00:00 2001
From: Eugene <junta.kristobal@gmail.com>
Date: Thu, 17 Mar 2022 05:39:54 +0100
Subject: [PATCH 571/716] Suggest installing the brotli package instead of
 brotlipy (#4267)

---
 docs/topics/downloader-middleware.rst | 5 +++--
 tests/requirements.txt                | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index a15637ed6..912600428 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -704,14 +704,15 @@ HttpCompressionMiddleware
    sent/received from web sites.
 
    This middleware also supports decoding `brotli-compressed`_ as well as
-   `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
+   `zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is
    installed, respectively.
 
 .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
-.. _brotlipy: https://pypi.org/project/brotlipy/
+.. _brotli: https://pypi.org/project/Brotli/
 .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
 .. _zstandard: https://pypi.org/project/zstandard/
 
+
 HttpCompressionMiddleware Settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 398d1d16d..d2a8aae1b 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -12,7 +12,7 @@ uvloop; platform_system != "Windows" and python_version > '3.6'
 
 # optional for shell wrapper tests
 bpython
-brotlipy  # optional for HTTP compress downloader middleware tests
+brotli  # optional for HTTP compress downloader middleware tests
 zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 ipython
 pywin32; sys_platform == "win32"

From 0905d42e33871e976760d880316210d4953cd5df Mon Sep 17 00:00:00 2001
From: Yann Defretin <kinoute@gmail.com>
Date: Thu, 17 Mar 2022 11:19:09 +0100
Subject: [PATCH 572/716] refactored url_has_any_extension function
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/utils/url.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index bae5a9433..4d5e9ae82 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -31,7 +31,8 @@ def url_is_from_spider(url, spider):
 
 def url_has_any_extension(url, extensions):
     """Return True if the url ends with one of the extensions provided"""
-    return any(parse_url(url).path.lower().endswith(ext) for ext in extensions)
+    lowercase_path = parse_url(url).path.lower()
+    return any(lowercase_path.endswith(ext) for ext in extensions)
 
 def parse_url(url, encoding=None):
     """Return urlparsed url from the given argument (which could be an already

From 6a3f2ee6876145bd4bd9ee1ff89d94474a1e85a0 Mon Sep 17 00:00:00 2001
From: FJMonteroInformatica <todofutbol024@gmail.com>
Date: Thu, 17 Mar 2022 20:09:56 +0100
Subject: [PATCH 573/716] HTML Conventions

---
 docs/_static/selectors-sample1.html           | 31 +++++++-------
 .../link_extractor/linkextractor.html         | 40 ++++++++++---------
 .../link_extractor/linkextractor_latin1.html  |  8 ++--
 .../link_extractor/linkextractor_no_href.html |  3 +-
 .../link_extractor/linkextractor_noenc.html   | 23 ++++++-----
 tests/sample_data/test_site/index.html        | 31 +++++++-------
 tests/sample_data/test_site/item1.html        | 27 ++++++-------
 tests/sample_data/test_site/item2.html        | 29 ++++++--------
 8 files changed, 96 insertions(+), 96 deletions(-)

diff --git a/docs/_static/selectors-sample1.html b/docs/_static/selectors-sample1.html
index 8a79a3381..915718832 100644
--- a/docs/_static/selectors-sample1.html
+++ b/docs/_static/selectors-sample1.html
@@ -1,16 +1,17 @@
-<html>
- <head>
-  <base href='http://example.com/' />
-  <title>Example website</title>
- </head>
- <body>
-  <div id='images'>
-   <a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' /></a>
-   <a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a>
-   <a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a>
-   <a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a>
-   <a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a>
-  </div>
- </body>
-</html>
+<!DOCTYPE html>
 
+<html>
+  <head>
+    <base href='http://example.com/' />
+    <title>Example website</title>
+  </head>
+  <body>
+    <div id='images'>
+      <a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' alt='image1'/></a>
+      <a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' alt='image2'/></a>
+      <a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' alt='image3'/></a>
+      <a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' alt='image4'/></a>
+      <a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' alt='image5'/></a>
+    </div>
+  </body>
+</html>
\ No newline at end of file
diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html
index 2307ea865..e3a2a4145 100644
--- a/tests/sample_data/link_extractor/linkextractor.html
+++ b/tests/sample_data/link_extractor/linkextractor.html
@@ -1,20 +1,22 @@
+<!DOCTYPE html>
+
 <html>
-<head>
-<base href='http://example.com' />
-<title>Sample page with links for testing LinkExtractor</title>
-</head>
-<body>
-<div id='wrapper'>
-<div id='subwrapper'>
-<area href='sample1.html' />
-<a href='sample2.html'>sample 2<img src='sample2.jpg'/></a>
-</div>
-<a href='http://example.com/sample3.html' title='sample 3'>sample 3 text</a>
-<a href='sample3.html'>sample 3 repetition</a>
-<a href='sample3.html#foo'>sample 3 repetition with fragment</a>
-<a href='http://www.google.com/something'></a>
-<a href='http://example.com/innertag.html'><b>inner</b> tag</a>
-<a href=' page 4.html '>href with whitespaces</a>
-</div>
-</body>
-</html>
+  <head>
+    <base href='http://example.com' />
+    <title>Sample page with links for testing LinkExtractor</title>
+  </head>
+  <body>
+    <div id='wrapper'>
+      <div id='subwrapper'>
+        <area href='sample1.html' alt='sample1'/>
+        <a href='sample2.html'>sample 2<img src='sample2.jpg' alt='sample2'/></a>
+      </div>
+      <a href='http://example.com/sample3.html' title='sample 3'>sample 3 text</a>
+      <a href='sample3.html'>sample 3 repetition</a>
+      <a href='sample3.html#foo'>sample 3 repetition with fragment</a>
+      <a href='http://www.google.com/something'></a>
+      <a href='http://example.com/innertag.html'><strong>inner</strong> tag</a>
+      <a href='page 4.html'>href with whitespaces</a>
+    </div>
+  </body>
+</html>
\ No newline at end of file
diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html
index e7eee18de..1e05bf0f0 100644
--- a/tests/sample_data/link_extractor/linkextractor_latin1.html
+++ b/tests/sample_data/link_extractor/linkextractor_latin1.html
@@ -1,3 +1,5 @@
+<!DOCTYPE html>
+
 <html>
   <head>
     <meta http-equiv="Content-Type" content="text/html; charset=latin-1">
@@ -7,11 +9,11 @@
 <body>
   <div id='wrapper'>
     <div id='subwrapper'>
-      <a href='sample_�.html'><img src='sample2.jpg'/></a>
+      <a href='sample_ñ.html'><img src='sample2.jpg' alt='sample2'/></a>
     </div>
-    <a href='sample_�.html' title='sample �'>sample � text</a>
+    <a href='sample_á.html' title='sample á'>sample á text</a>
     <div id='subwrapper2'>
-      <a href='sample_�.html?price=�32&�=unit'><img src='sample3.jpg'/></a>
+      <a href='sample_ö.html?price=£32&µ=unit'><img src='sample3.jpg' alt='sample3'/></a>
     </div>
   </div>
 </body>
diff --git a/tests/sample_data/link_extractor/linkextractor_no_href.html b/tests/sample_data/link_extractor/linkextractor_no_href.html
index 0b01cede8..2d67ec6ff 100644
--- a/tests/sample_data/link_extractor/linkextractor_no_href.html
+++ b/tests/sample_data/link_extractor/linkextractor_no_href.html
@@ -1,3 +1,5 @@
+<!DOCTYPE html>
+
 <html>
   <head>
     <base href='http://example.com' />
@@ -21,5 +23,4 @@
     </div>
   </div>
 </body>
-
 </html>
\ No newline at end of file
diff --git a/tests/sample_data/link_extractor/linkextractor_noenc.html b/tests/sample_data/link_extractor/linkextractor_noenc.html
index f9166adbe..6fa137cd9 100644
--- a/tests/sample_data/link_extractor/linkextractor_noenc.html
+++ b/tests/sample_data/link_extractor/linkextractor_noenc.html
@@ -1,14 +1,17 @@
+<!DOCTYPE html>
+
 <html>
-<head>
-<base href='http://example.com' />
-<title>Sample page without encoding for testing LinkExtractor</title>
-</head>
+  <head>
+    <base href='http://example.com' />
+    <title>Sample page without encoding for testing LinkExtractor</title>
+  </head>
+
 <body>
-<div id='wrapper'>
-<div id='subwrapper'>
-<a href='sample_ñ.html'><img src='sample2.jpg'/></a>
-</div>
-<a href='sample_€.html' title='sample €'>sample € text</a>
-</div>
+  <div id='wrapper'>
+    <div id='subwrapper'>
+      <a href='sample_ñ.html'><img src='sample2.jpg' alt='sample2'/></a>
+    </div>
+    <a href='sample_€.html' title='sample €'>sample € text</a>
+  </div>
 </body>
 </html>
diff --git a/tests/sample_data/test_site/index.html b/tests/sample_data/test_site/index.html
index d268c846a..afe17d8e2 100644
--- a/tests/sample_data/test_site/index.html
+++ b/tests/sample_data/test_site/index.html
@@ -1,18 +1,15 @@
+<!DOCTYPE html>
+
 <html>
-
-<head>
-<title>Scrapy test site</title>
-</head>
-
-<body>
-
-<h1>Scrapy test site</h1>
-
-<ul>
-<li><a href="item1.html">Item 1</li>
-<li><a href="item2.html">Item 2</li>
-<li><a href="item999.html">Item 999 (not found)</li>
-</ul>
-
-</body>
-</html>
+  <head>
+    <title>Scrapy test site</title>
+  </head>
+  <body>
+    <h1>Scrapy test site</h1>
+    <ul>
+      <li><a href="item1.html">Item 1</a></li>
+      <li><a href="item2.html">Item 2</a></li>
+      <li><a href="item999.html">Item 999 (not found)</a></li>
+    </ul>
+  </body>
+</html>
\ No newline at end of file
diff --git a/tests/sample_data/test_site/item1.html b/tests/sample_data/test_site/item1.html
index ceeb6dc87..ee39f16f3 100644
--- a/tests/sample_data/test_site/item1.html
+++ b/tests/sample_data/test_site/item1.html
@@ -1,17 +1,14 @@
+<!DOCTYPE html>
+
 <html>
-
-<head>
-<title>Item 1 - Scrapy test site</title>
-</head>
-
-<body>
-
-<h1>Item 1 name</h1>
-
-<ul>
-<li>Price: $100</li>
-<li>Stock: 12</li>
-</ul>
-
-</body>
+  <head>
+    <title>Item 1 - Scrapy test site</title>
+  </head>
+    <body>
+      <h1>Item 1 name</h1>
+      <ul>
+        <li>Price: $100</li>
+        <li>Stock: 12</li>
+      </ul>
+    </body>
 </html>
diff --git a/tests/sample_data/test_site/item2.html b/tests/sample_data/test_site/item2.html
index a64c92810..f40f70750 100644
--- a/tests/sample_data/test_site/item2.html
+++ b/tests/sample_data/test_site/item2.html
@@ -1,17 +1,14 @@
+<!DOCTYPE html>
+
 <html>
-
-<head>
-<title>Item 2 - Scrapy test site</title>
-</head>
-
-<body>
-
-<h1>Item 2 name</h1>
-
-<ul>
-<li>Price: $200</li>
-<li>Stock: 5</li>
-</ul>
-
-</body>
-</html>
+  <head>
+    <title>Item 2 - Scrapy test site</title>
+  </head>
+  <body>
+    <h1>Item 2 name</h1>
+    <ul>
+      <li>Price: $200</li>
+      <li>Stock: 5</li>
+    </ul>
+  </body>
+</html>
\ No newline at end of file

From b95c634b861bacc2b2ee3beeb5fcf64ab8607eea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 17 Mar 2022 22:23:25 +0100
Subject: [PATCH 574/716] Document how to enforce Scrapy versions on Scrapy
 components

---
 docs/index.rst                    | 13 +++--
 docs/topics/asyncio.rst           | 24 +++++++++
 docs/topics/components.rst        | 84 +++++++++++++++++++++++++++++++
 docs/topics/coroutines.rst        |  6 +--
 docs/topics/spider-middleware.rst |  8 +--
 5 files changed, 125 insertions(+), 10 deletions(-)
 create mode 100644 docs/topics/components.rst

diff --git a/docs/index.rst b/docs/index.rst
index 75e08f537..6e22db884 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -229,10 +229,11 @@ Extending Scrapy
    topics/downloader-middleware
    topics/spider-middleware
    topics/extensions
-   topics/api
    topics/signals
    topics/scheduler
    topics/exporters
+   topics/components
+   topics/api
 
 
 :doc:`topics/architecture`
@@ -247,9 +248,6 @@ Extending Scrapy
 :doc:`topics/extensions`
     Extend Scrapy with your custom functionality
 
-:doc:`topics/api`
-    Use it on extensions and middlewares to extend Scrapy functionality
-
 :doc:`topics/signals`
     See all available signals and how to work with them.
 
@@ -259,6 +257,13 @@ Extending Scrapy
 :doc:`topics/exporters`
     Quickly export your scraped items to a file (XML, CSV, etc).
 
+:doc:`topics/components`
+    Learn the common API and some good practices when building custom Scrapy
+    components.
+
+:doc:`topics/api`
+    Use it on extensions and middlewares to extend Scrapy functionality
+
 
 All the rest
 ============
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 3a6941a2c..dbee7146d 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -96,3 +96,27 @@ Futures. Scrapy provides two helpers for this:
          down to Scrapy 2.0 (earlier versions do not support
          :mod:`asyncio`), you can copy the implementation of these functions
          into your own code.
+
+
+.. _enforce-asyncio-requirement:
+
+Enforcing asyncio as a requirement
+==================================
+
+If you are writing a :ref:`component <topics-components>` that requires asyncio
+to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to
+:ref:`enforce it as a requirement <enforce-component-requirements>`. For
+example::
+
+    from scrapy.utils.reactor import is_asyncio_reactor_installed
+
+    class MyComponent:
+
+        def __init__(self):
+            if not is_asyncio_reactor_installed():
+                raise ValueError(
+                    f"{MyComponent.__qualname__} requires the asyncio Twisted "
+                    f"reactor. Make sure you have it configured in the "
+                    f"TWISTED_REACTOR setting. See the asyncio documentation "
+                    f"of Scrapy for more information."
+                )
diff --git a/docs/topics/components.rst b/docs/topics/components.rst
new file mode 100644
index 000000000..1fff2d61a
--- /dev/null
+++ b/docs/topics/components.rst
@@ -0,0 +1,84 @@
+.. _topics-components:
+
+==========
+Components
+==========
+
+A Scrapy component is any class whose objects are created using
+:func:`scrapy.utils.misc.create_instance`.
+
+That includes the classes that you may assign to the following settings:
+
+-   :setting:`DNS_RESOLVER`
+
+-   :setting:`DOWNLOAD_HANDLERS`
+
+-   :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`
+
+-   :setting:`DOWNLOADER_MIDDLEWARES`
+
+-   :setting:`DUPEFILTER_CLASS`
+
+-   :setting:`EXTENSIONS`
+
+-   :setting:`FEED_EXPORTERS`
+
+-   :setting:`FEED_STORAGES`
+
+-   :setting:`ITEM_PIPELINES`
+
+-   :setting:`SCHEDULER`
+
+-   :setting:`SCHEDULER_DISK_QUEUE`
+
+-   :setting:`SCHEDULER_MEMORY_QUEUE`
+
+-   :setting:`SCHEDULER_PRIORITY_QUEUE`
+
+-   :setting:`SPIDER_MIDDLEWARES`
+
+Third-party Scrapy components may also let you define additional Scrapy
+components, usually configurable through :ref:`settings <topics-settings>`, to
+modify their behavior.
+
+.. _enforce-component-requirements:
+
+Enforcing component requirements
+================================
+
+Sometimes, your components may only be intended to work under certain
+conditions. For example, the may require a minimum version of Scrapy to work as
+intended, or they may require certain settings to have specific values.
+
+In addition to describing those conditions in the documentation of your
+component, it is a good practice to raise an exception from the ``__init__``
+method of your component if those conditions are not met at run time.
+
+In the case of :ref:`downloader middlewares <topics-downloader-middleware>`,
+:ref:`extensions <topics-extensions>`, :ref:`item pipelines
+<topics-item-pipeline>`, and :ref:`spider middlewares
+<topics-spider-middleware>`, you should raise
+:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a
+parameter to the exception so that it is printed in the logs, for the user to
+see. For other components, feel free to raise whatever other exception feels
+right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy
+version mismatch, while :exc:`ValueError` may be better if the issue is the
+value of a setting.
+
+If your requirement is a minimum Scrapy version, you may use
+:attr:`scrapy.__version__` to enforce your requirement. For example::
+
+    from pkg_resources import parse_version
+
+    import scrapy
+
+    class MyComponent:
+
+        def __init__(self):
+            if parse_version(scrapy.__version__) < parse_version('VERSION'):
+                raise RuntimeError(
+                    f"{MyComponent.__qualname__} requires Scrapy VERSION or "
+                    f"later, which allow defining the process_spider_output "
+                    f"method of spider middlewares as an asynchronous "
+                    f"generator."
+                )
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index efc4566a0..55d013c06 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -162,7 +162,7 @@ Asynchronous-to-synchronous conversions are supported for backward
 compatibility, but they are deprecated and will stop working in a future
 version of Scrapy.
 
-To avoid asynchronous-to-synchronous conversion, when defining ``Request``
+To avoid asynchronous-to-synchronous conversions, when defining ``Request``
 callbacks as coroutine methods or when using spider middlewares whose
 ``process_spider_output`` method is an :term:`asynchronous generator`, all
 active spider middlewares must either have their ``process_spider_output``
@@ -177,8 +177,8 @@ process_spider_output_async method <universal-spider-middleware>`.
 
 .. _universal-spider-middleware:
 
-Universal spider middleware
-===========================
+Universal spider middlewares
+============================
 
 .. versionadded:: VERSION
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 787545ed2..816cb5e03 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -111,9 +111,11 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
         Consider defining this method as an :term:`asynchronous generator`,
         which will be a requirement in a future version of Scrapy. However, if
-        you wish your spider middleware to work with Scrapy versions earlier
-        than Scrapy VERSION, :ref:`make your spider middleware universal
-        <universal-spider-middleware>` instead.
+        you plan on sharing your spider middleware with other people, consider
+        either :ref:`enforcing Scrapy VERSION <enforce-component-requirements>`
+        as a minimum requirement of your spider middleware, or :ref:`making
+        your spider middleware universal <universal-spider-middleware>` so that
+        it works with Scrapy versions earlier than Scrapy VERSION.
 
         :param response: the response which generated this output from the
           spider

From fcf3d8e0a0df447fd7cd81e98c846a16b8b42a73 Mon Sep 17 00:00:00 2001
From: D00399830 <d00399830@dmail.dixie.edu>
Date: Mon, 21 Mar 2022 14:09:31 -0600
Subject: [PATCH 575/716] Updated the documentation for developer tools to have
 JavaScript instead of Javascript, as JavaScript is the more correct way to
 write it

---
 docs/topics/developer-tools.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index 96475899f..9bf97c628 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -19,14 +19,14 @@ Caveats with inspecting the live browser DOM
 
 Since Developer Tools operate on a live browser DOM, what you'll actually see
 when inspecting the page source is not the original HTML, but a modified one
-after applying some browser clean up and executing Javascript code.  Firefox,
+after applying some browser clean up and executing JavaScript code.  Firefox,
 in particular, is known for adding ``<tbody>`` elements to tables.  Scrapy, on
 the other hand, does not modify the original page HTML, so you won't be able to
 extract any data if you use ``<tbody>`` in your XPath expressions.
 
 Therefore, you should keep in mind the following things:
 
-* Disable Javascript while inspecting the DOM looking for XPaths to be
+* Disable JavaScript while inspecting the DOM looking for XPaths to be
   used in Scrapy (in the Developer Tools settings click `Disable JavaScript`)
 
 * Never use full XPath paths, use relative and clever ones based on attributes

From 2227be7af6d0504d52bd4c2e299efb1becbd992f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?V=C3=ADctor=20Ruiz?= <victormruizs@gmail.com>
Date: Tue, 22 Mar 2022 15:21:16 +0100
Subject: [PATCH 576/716] Fix a typo in the HTTP cache documentation (#5455)

---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 912600428..29e350651 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -366,7 +366,7 @@ HttpCacheMiddleware
     This middleware provides low-level cache to all HTTP requests and responses.
     It has to be combined with a cache storage backend as well as a cache policy.
 
-    Scrapy ships with three HTTP cache storage backends:
+    Scrapy ships with the following HTTP cache storage backends:
 
         * :ref:`httpcache-storage-fs`
         * :ref:`httpcache-storage-dbm`

From 4af22bf157a5d25703cf5142097ba1c277f0d0d4 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Fri, 8 Apr 2022 14:26:23 +0500
Subject: [PATCH 577/716] Pin mitmproxy to < 8 for now (#5459)

---
 tox.ini | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index fcd3563b2..aba94d79d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,10 +12,11 @@ deps =
     -rtests/requirements.txt
     # mitmproxy does not support PyPy
     # mitmproxy does not support Windows when running Python < 3.7
-    # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe
+    # Python 3.9+ requires mitmproxy >= 5.3.0
     # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
     #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
-    mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
+    # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
+    mitmproxy >= 4.0.4, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
     mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
     markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy'

From bae3f8745589b09e0ea75fa463a5423546ca442c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 8 Apr 2022 12:04:02 +0200
Subject: [PATCH 578/716] Cover a backward-incompatible Request serialization
 change in the 2.6 release notes

---
 docs/news.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 5d92067b5..e4e2bce3c 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -113,6 +113,9 @@ Backward-incompatible changes
     meet expectations, :exc:`TypeError` is now raised at startup time. Before,
     other exceptions would be raised at run time. (:issue:`3559`)
 
+-   The ``_encoding`` field of serialized :class:`~scrapy.http.Request` objects
+    is now named ``encoding``, in line with all other fields (:issue:`5130`)
+
 
 Deprecation removals
 ~~~~~~~~~~~~~~~~~~~~

From 319e67f779163df3ad44327bfbc9733edcb34908 Mon Sep 17 00:00:00 2001
From: Yash <76577754+yash-fn@users.noreply.github.com>
Date: Sat, 26 Mar 2022 18:17:03 -0500
Subject: [PATCH 579/716] documentation update for multiple spiders

i noticed passing settings to configure logging function made weird output go away. checked documentation and it says first parameter is settings file. Is this correct?
---
 docs/topics/practices.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index d0207fd18..7313c9246 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -180,8 +180,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
         # Your second spider definition
         ...
 
-    configure_logging()
     settings = get_project_settings()
+    configure_logging(settings)
     runner = CrawlerRunner(settings)
 
     @defer.inlineCallbacks

From aead27bcbdf7c2a4d959dcb357c7f12cc8411739 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 14 Apr 2022 15:06:22 +0200
Subject: [PATCH 580/716] Add release notes for 2.6.2 (#5448)

---
 docs/news.rst | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index e4e2bce3c..2e0b43455 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,23 @@
 Release notes
 =============
 
+.. _release-2.6.2:
+
+Scrapy 2.6.2 (2022-03-15)
+-------------------------
+
+Fixes additional regressions introduced in 2.6.0:
+
+-   :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple
+    spiders (:issue:`5435`, :issue:`5436`)
+
+-   Fixed an exception that was being logged after the spider finished under
+    certain conditions (:issue:`5437`, :issue:`5440`)
+
+-   The ``--output``/``-o`` command-line parameter supports again a value
+    starting with a hyphen (:issue:`5444`, :issue:`5445`)
+
+
 .. _release-2.6.1:
 
 Scrapy 2.6.1 (2022-03-01)

From 636127ec1ea2b8949438015c2167ab5d009ff1bf Mon Sep 17 00:00:00 2001
From: Alex <alexpdev@protonmail.com>
Date: Sun, 17 Apr 2022 12:01:24 -0700
Subject: [PATCH 581/716] tests that all CLI help messages don't throw errors

---
 tests/test_commands.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index b5e6c2b8b..76d5f3935 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -930,3 +930,17 @@ class MySpider(scrapy.Spider):
         args = ['-o', 'example1.json', '-O', 'example2.json']
         log = self.get_log(spider_code, args=args)
         self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
+
+
+class HelpMessageTest(CommandTest):
+
+    def setUp(self):
+        super().setUp()
+        self.commands = ["parse", "startproject", "view", "crawl", "edit",
+                         "list", "fetch", "settings", "shell", "runspider",
+                         "version", "genspider", "check", "bench"]
+
+    def test_help_messages(self):
+        for command in self.commands:
+            _, out, _ = self.proc(command, "-h")
+            self.assertIn("Usage", out)

From b0f5503cb8d0590aab4d9c91b0ee660d98d1e4a6 Mon Sep 17 00:00:00 2001
From: Alex <alexpdev@protonmail.com>
Date: Sat, 16 Apr 2022 18:17:47 -0700
Subject: [PATCH 582/716] Fixes Issue #5481

---
 scrapy/commands/parse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index a3f6b96f4..6365fbdd0 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -51,7 +51,7 @@ class Command(BaseRunSpiderCommand):
         parser.add_argument("--cbkwargs", dest="cbkwargs",
                             help="inject extra callback kwargs into the Request, it must be a valid raw json string")
         parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
-                            help="maximum depth for parsing requests [default: %default]")
+                            help=f"maximum depth for parsing requests [default: {self.max_level}]")
         parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
                             help="print each depth level one by one")
 

From 56c9098d6af2a57cc11927959d47253999cddd46 Mon Sep 17 00:00:00 2001
From: Alex <alexpdev@protonmail.com>
Date: Sat, 16 Apr 2022 18:33:12 -0700
Subject: [PATCH 583/716] changed default depth to 1

---
 scrapy/commands/parse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 6365fbdd0..a798ef945 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -51,7 +51,7 @@ class Command(BaseRunSpiderCommand):
         parser.add_argument("--cbkwargs", dest="cbkwargs",
                             help="inject extra callback kwargs into the Request, it must be a valid raw json string")
         parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
-                            help=f"maximum depth for parsing requests [default: {self.max_level}]")
+                            help="maximum depth for parsing requests [default: 1]")
         parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
                             help="print each depth level one by one")
 

From 915c288205e2b9a0bdbbe18cc67cd23ba5bb4de3 Mon Sep 17 00:00:00 2001
From: Alex <alexpdev@protonmail.com>
Date: Sun, 17 Apr 2022 10:49:50 -0700
Subject: [PATCH 584/716] edit

---
 scrapy/commands/parse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index a798ef945..8e52d0d76 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -51,7 +51,7 @@ class Command(BaseRunSpiderCommand):
         parser.add_argument("--cbkwargs", dest="cbkwargs",
                             help="inject extra callback kwargs into the Request, it must be a valid raw json string")
         parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
-                            help="maximum depth for parsing requests [default: 1]")
+                            help="maximum depth for parsing requests [default: %(default)s]")
         parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
                             help="print each depth level one by one")
 

From 7de9ed5bd42052f6491c939513adde6f6243ce2f Mon Sep 17 00:00:00 2001
From: PluT00 <314lut00@gmail.com>
Date: Sun, 1 May 2022 01:34:35 +0300
Subject: [PATCH 585/716] Deprecate scrapy.pipelines.images.NoimagesDrop

---
 scrapy/pipelines/images.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 9c99dc69e..df4575a41 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -15,10 +15,12 @@ from scrapy.http import Request
 from scrapy.pipelines.files import FileException, FilesPipeline
 # TODO: from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
+from scrapy.utils.decorators import deprecated
 from scrapy.utils.misc import md5sum
 from scrapy.utils.python import to_bytes
 
 
+@deprecated()
 class NoimagesDrop(DropItem):
     """Product with no images exception"""
 

From 9f659bd63c04656f1c0601d6338f9050d5049fe8 Mon Sep 17 00:00:00 2001
From: PluT00 <314lut00@gmail.com>
Date: Sun, 1 May 2022 13:36:15 +0300
Subject: [PATCH 586/716] Fix deprecation of
 scrapy.pipelines.images.NoimagesDrop

---
 scrapy/pipelines/images.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index df4575a41..c7a04a0cf 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -5,25 +5,28 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+import warnings
 from contextlib import suppress
 from io import BytesIO
 
 from itemadapter import ItemAdapter
 
-from scrapy.exceptions import DropItem, NotConfigured
+from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.pipelines.files import FileException, FilesPipeline
 # TODO: from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
-from scrapy.utils.decorators import deprecated
 from scrapy.utils.misc import md5sum
 from scrapy.utils.python import to_bytes
 
 
-@deprecated()
 class NoimagesDrop(DropItem):
     """Product with no images exception"""
 
+    def __init__(self, *args, **kwargs):
+        warnings.warn("The NoimagesDrop class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2)
+        super().__init__(*args, **kwargs)
+
 
 class ImageException(FileException):
     """General image error exception"""

From cc16af35af8b20e04b42ca84431820b2b092f379 Mon Sep 17 00:00:00 2001
From: PluT00 <314lut00@gmail.com>
Date: Tue, 3 May 2022 11:29:21 +0300
Subject: [PATCH 587/716] Add deprecation warning test for
 scrapy.pipelines.images.NoimagesDrop

---
 tests/test_pipeline_images.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index c69cd0e4a..613190f9c 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -4,14 +4,16 @@ import random
 from shutil import rmtree
 from tempfile import mkdtemp
 from unittest import skipIf
+from warnings import catch_warnings
 
 import attr
 from itemadapter import ItemAdapter
 from twisted.trial import unittest
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.item import Field, Item
-from scrapy.pipelines.images import ImagesPipeline
+from scrapy.pipelines.images import ImagesPipeline, NoimagesDrop
 from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
 
@@ -413,6 +415,22 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
                              expected_value)
 
 
+class NoimagesDropTestCase(unittest.TestCase):
+
+    def test_deprecation_warning(self):
+        arg = str()
+        with catch_warnings(record=True) as warnings:
+            NoimagesDrop(arg)
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+        with catch_warnings(record=True) as warnings:
+            class SubclassedNoimagesDrop(NoimagesDrop):
+                pass
+            SubclassedNoimagesDrop(arg)
+            self.assertEqual(len(warnings), 1)
+            self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
+
+
 def _create_image(format, *a, **kw):
     buf = io.BytesIO()
     Image.new(*a, **kw).save(buf, format)

From b2afcbfe2bf090827540d072866bef0d1ab3a3e8 Mon Sep 17 00:00:00 2001
From: AngelikiBoura <73474686+AngelikiBoura@users.noreply.github.com>
Date: Thu, 5 May 2022 16:49:52 +0300
Subject: [PATCH 588/716] Fix typos in three files for Flake8 check (#5487)

* Fix typos in extensions files

Made some fixes in files memusage.py and statsmailer.py
in order to pass the flake8 check.

* Fix typos in twisted_reactor_custom_settings_same.py

A small change was needed in order for flake8 check
to pass.
---
 scrapy/extensions/memusage.py                          | 10 +++++-----
 scrapy/extensions/statsmailer.py                       |  1 +
 .../twisted_reactor_custom_settings_same.py            |  1 +
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 9de119a10..f5081a7d7 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -33,8 +33,8 @@ class MemoryUsage:
         self.crawler = crawler
         self.warned = False
         self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
-        self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
-        self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
+        self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
+        self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
         self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
         self.mail = MailSender.from_settings(crawler.settings)
         crawler.signals.connect(self.engine_started, signal=signals.engine_started)
@@ -77,7 +77,7 @@ class MemoryUsage:
     def _check_limit(self):
         if self.get_virtual_size() > self.limit:
             self.crawler.stats.set_value('memusage/limit_reached', 1)
-            mem = self.limit/1024/1024
+            mem = self.limit / 1024 / 1024
             logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
                          {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
@@ -94,11 +94,11 @@ class MemoryUsage:
                 self.crawler.stop()
 
     def _check_warning(self):
-        if self.warned: # warn only once
+        if self.warned:  # warn only once
             return
         if self.get_virtual_size() > self.warning:
             self.crawler.stats.set_value('memusage/warning_reached', 1)
-            mem = self.warning/1024/1024
+            mem = self.warning / 1024 / 1024
             logger.warning("Memory usage reached %(memusage)dM",
                            {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index bcdbaff24..739e6b958 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -8,6 +8,7 @@ from scrapy import signals
 from scrapy.mail import MailSender
 from scrapy.exceptions import NotConfigured
 
+
 class StatsMailer:
 
     def __init__(self, stats, recipients, mail):
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
index 1f5a44010..72bb986bc 100644
--- a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
@@ -8,6 +8,7 @@ class AsyncioReactorSpider1(scrapy.Spider):
         "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
     }
 
+
 class AsyncioReactorSpider2(scrapy.Spider):
     name = 'asyncio_reactor2'
     custom_settings = {

From 83c1939281197242511931c6e9f356f2498eb623 Mon Sep 17 00:00:00 2001
From: Andreas Tziortziortziopoulos <t8170132@dias.aueb.gr>
Date: Fri, 6 May 2022 03:59:30 +0300
Subject: [PATCH 589/716] Issue #3264, fix error handling when spider is not
 matched

Changes
Implementation:
- Check whether Spider exists or is None, and if it's None skip execution of start_requests() with non existing Spider
Testing:
- Add a test case with invalid url inside test_command_parse
  Test proves that non-matched Spider does not throw an AttributeError
---
 scrapy/commands/parse.py    | 3 ++-
 tests/test_command_parse.py | 5 +++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index a3f6b96f4..99fc8f955 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -146,7 +146,8 @@ class Command(BaseRunSpiderCommand):
 
         def _start_requests(spider):
             yield self.prepare_request(spider, Request(url), opts)
-        self.spidercls.start_requests = _start_requests
+        if self.spidercls:
+            self.spidercls.start_requests = _start_requests
 
     def start_parsing(self, url, opts):
         self.crawler_process.crawl(self.spidercls, **opts.spargs)
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index f21ee971d..0622074a3 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,6 +1,7 @@
 import os
 import argparse
 from os.path import join, abspath, isfile, exists
+
 from twisted.internet import defer
 from scrapy.commands import parse
 from scrapy.settings import Settings
@@ -222,6 +223,10 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
 
+        status, out, stderr = yield self.execute([self.url('/invalid_url')])
+        self.assertEqual(status, 0)
+        self.assertIn("""""", _textmode(stderr))
+
     @defer.inlineCallbacks
     def test_output_flag(self):
         """Checks if a file was created successfully having

From 84c29a286f6b9bc94a5318ca68cd4fc21244443a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 20 May 2022 06:45:38 +0200
Subject: [PATCH 590/716] Unset the release date of still-unreleased 2.6.2
 (#5503)

---
 docs/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 2e0b43455..ca9aeb783 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5,7 +5,7 @@ Release notes
 
 .. _release-2.6.2:
 
-Scrapy 2.6.2 (2022-03-15)
+Scrapy 2.6.2 (2022-0?-??)
 -------------------------
 
 Fixes additional regressions introduced in 2.6.0:

From 1c1cd5d8eae48eade88560426499846d217a555f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 20 May 2022 07:05:26 +0200
Subject: [PATCH 591/716] Update the 2.6.2 release notes

---
 docs/news.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index ca9aeb783..ffeb50390 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -19,6 +19,9 @@ Fixes additional regressions introduced in 2.6.0:
 -   The ``--output``/``-o`` command-line parameter supports again a value
     starting with a hyphen (:issue:`5444`, :issue:`5445`)
 
+-   The ``scrapy parse -h`` command no longer throws an error (:issue:`5481`,
+    :issue:`5482`)
+
 
 .. _release-2.6.1:
 

From 965fde24a4798ee51f05ce8669b7a28958ad3238 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Fri, 8 Apr 2022 14:26:23 +0500
Subject: [PATCH 592/716] Pin mitmproxy to < 8 for now (#5459)

---
 tox.ini | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index db151f215..d13bb7b38 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,10 +12,11 @@ deps =
     -rtests/requirements.txt
     # mitmproxy does not support PyPy
     # mitmproxy does not support Windows when running Python < 3.7
-    # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe
+    # Python 3.9+ requires mitmproxy >= 5.3.0
     # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
     #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
-    mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
+    # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
+    mitmproxy >= 4.0.4, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
     mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
     markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy'

From 078622cfb0ee364acba5d91a20244f9c1ee87d30 Mon Sep 17 00:00:00 2001
From: Maxime Nannan <28675918+mnannan@users.noreply.github.com>
Date: Fri, 20 May 2022 08:30:06 +0200
Subject: [PATCH 593/716] Fix file expiration issue with GCS (#5318)

---
 scrapy/pipelines/files.py    | 10 +++++++---
 tests/test_pipeline_files.py | 23 +++++++++++++++++++++++
 tox.ini                      |  7 ++++---
 3 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 5c52c6c28..906e7eb24 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -222,8 +222,8 @@ class GCSFilesStore:
                 return {'checksum': checksum, 'last_modified': last_modified}
             else:
                 return {}
-
-        return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
+        blob_path = self._get_blob_path(path)
+        return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess)
 
     def _get_content_type(self, headers):
         if headers and 'Content-Type' in headers:
@@ -231,8 +231,12 @@ class GCSFilesStore:
         else:
             return 'application/octet-stream'
 
+    def _get_blob_path(self, path):
+        return self.prefix + path
+
     def persist_file(self, path, buf, info, meta=None, headers=None):
-        blob = self.bucket.blob(self.prefix + path)
+        blob_path = self._get_blob_path(path)
+        blob = self.bucket.blob(blob_path)
         blob.cache_control = self.CACHE_CONTROL
         blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
         return threads.deferToThread(
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 4e1b90787..0ff2045ed 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -525,6 +525,29 @@ class TestGCSFilesStore(unittest.TestCase):
         self.assertEqual(blob.content_type, 'application/octet-stream')
         self.assertIn(expected_policy, acl)
 
+    @defer.inlineCallbacks
+    def test_blob_path_consistency(self):
+        """Test to make sure that paths used to store files is the same as the one used to get
+        already uploaded files.
+        """
+        assert_gcs_environ()
+        try:
+            import google.cloud.storage # noqa
+        except ModuleNotFoundError:
+            raise unittest.SkipTest("google-cloud-storage is not installed")
+        else:
+            with mock.patch('google.cloud.storage') as _:
+                with mock.patch('scrapy.pipelines.files.time') as _:
+                    uri = 'gs://my_bucket/my_prefix/'
+                    store = GCSFilesStore(uri)
+                    store.bucket = mock.Mock()
+                    path = 'full/my_data.txt'
+                    yield store.persist_file(path, mock.Mock(), info=None, meta=None, headers=None)
+                    yield store.stat_file(path, info=None)
+                    expected_blob_path = store.prefix + path
+                    store.bucket.blob.assert_called_with(expected_blob_path)
+                    store.bucket.get_blob.assert_called_with(expected_blob_path)
+
 
 class TestFTPFileStore(unittest.TestCase):
     @defer.inlineCallbacks
diff --git a/tox.ini b/tox.ini
index d13bb7b38..6951b6d16 100644
--- a/tox.ini
+++ b/tox.ini
@@ -126,13 +126,14 @@ setenv =
 deps =
     {[testenv]deps}
     boto
+    google-cloud-storage
+    # Twisted[http2] currently forces old mitmproxy because of h2 version
+    # restrictions in their deps, so we need to pin old markupsafe here too.
+    markupsafe < 2.1.0
     reppy
     robotexclusionrulesparser
     Pillow>=4.0.0
     Twisted[http2]>=17.9.0
-    # Twisted[http2] currently forces old mitmproxy because of h2 version restrictions in their deps,
-    # so we need to pin old markupsafe here too
-    markupsafe < 2.1.0
 
 [testenv:asyncio]
 commands =

From b5c15d87ff5770220bca31792c89e58804b923bb Mon Sep 17 00:00:00 2001
From: Andreas Tziortziortziopoulos <t8170132@dias.aueb.gr>
Date: Sun, 22 May 2022 12:19:20 +0300
Subject: [PATCH 594/716] [issue3264] Separate test for not matched spider to a
 url

---
 tests/test_command_parse.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 0622074a3..0d992be56 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -223,9 +223,10 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
 
+    @defer.inlineCallbacks
+    def test_crawlspider_not_exists_with_not_matched_url(self):
         status, out, stderr = yield self.execute([self.url('/invalid_url')])
         self.assertEqual(status, 0)
-        self.assertIn("""""", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_output_flag(self):

From 86331900125dc311223cbd1ebb0e10d09e7c592d Mon Sep 17 00:00:00 2001
From: Mohammadtaher Abbasi <mohammadtaher.abbasi@zoodfood.com>
Date: Tue, 24 May 2022 14:47:00 +0430
Subject: [PATCH 595/716] pass on item to thumb_path function as additional
 argument resolves #5504

---
 scrapy/pipelines/images.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 9c99dc69e..45ac03820 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -141,7 +141,7 @@ class ImagesPipeline(FilesPipeline):
         yield path, image, buf
 
         for thumb_id, size in self.thumbs.items():
-            thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
+            thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)
             thumb_image, thumb_buf = self.convert_image(image, size)
             yield thumb_path, thumb_image, thumb_buf
 
@@ -179,6 +179,6 @@ class ImagesPipeline(FilesPipeline):
         image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return f'full/{image_guid}.jpg'
 
-    def thumb_path(self, request, thumb_id, response=None, info=None):
+    def thumb_path(self, request, thumb_id, response=None, info=None, item=None):
         thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return f'thumbs/{thumb_id}/{thumb_guid}.jpg'

From f39def4492f838b2414324e22a12f3b355f5c062 Mon Sep 17 00:00:00 2001
From: Mohammadtaher Abbasi <mohammadtaher.abbasi@zoodfood.com>
Date: Wed, 25 May 2022 23:57:38 +0430
Subject: [PATCH 596/716] add docs

---
 docs/topics/media-pipeline.rst | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 7dff78390..2513faae2 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -656,6 +656,26 @@ See here the methods that you can override in your custom Images Pipeline:
       .. versionadded:: 2.4
          The *item* parameter.
 
+   .. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)
+
+      This method is called for every item of  :setting:`IMAGES_THUMBS` per downloaded item. It returns the
+      thumbnail download path of the image originating from the specified
+      :class:`response <scrapy.http.Response>`.
+
+      In addition to ``response``, this method receives the original
+      :class:`request <scrapy.Request>`,
+      ``thumb_id``,
+      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
+      :class:`item <scrapy.Item>`.
+
+      You can override this method to customize the thumbnail download path of each image.
+      You can use the ``item`` to determine the file path based on some item
+      property.
+
+      By default the :meth:`thumb_path` method returns
+      ``thumbs/<size name>/<request URL hash>.<extension>``.
+
+
    .. method:: ImagesPipeline.get_media_requests(item, info)
 
       Works the same way as :meth:`FilesPipeline.get_media_requests` method,

From 5c586d78f0e1c5b66358ed644bb6e528ad4b062b Mon Sep 17 00:00:00 2001
From: Mohammadtaher Abbasi <mohammadtaher.abbasi@zoodfood.com>
Date: Wed, 25 May 2022 23:58:09 +0430
Subject: [PATCH 597/716] add tests

---
 tests/test_pipeline_images.py | 16 ++++++++++++++++
 tests/test_pipeline_media.py  | 28 +++++++++++++++++++++++++---
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index c69cd0e4a..dd94d296b 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -93,6 +93,22 @@ class ImagesPipelineTestCase(unittest.TestCase):
                                     info=object()),
                          'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg')
 
+    def test_thumbnail_name_from_item(self):
+        """
+        Custom thumbnail name based on item data, overriding default implementation
+        """
+
+        class CustomImagesPipeline(ImagesPipeline):
+            def thumb_path(self, request, thumb_id, response=None, info=None, item=None):
+                return f"thumb/{thumb_id}/{item.get('path')}"
+
+        thumb_path = CustomImagesPipeline.from_settings(Settings(
+            {'IMAGES_STORE': self.tempdir}
+        )).thumb_path
+        item = dict(path='path-to-store-file')
+        request = Request("http://example.com")
+        self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file')
+
     def test_convert_image(self):
         SIZE = (100, 100)
         # straigh forward case: RGB and JPEG
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 893d43052..a802c7cf1 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,4 +1,5 @@
 from typing import Optional
+import io
 
 from testfixtures import LogCapture
 from twisted.trial import unittest
@@ -355,9 +356,12 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
 
     def get_media_requests(self, item, info):
         item_url = item['image_urls'][0]
+        output_img = io.BytesIO()
+        img = Image.new('RGB', (60, 30), color='red')
+        img.save(output_img, format='JPEG')
         return Request(
             item_url,
-            meta={'response': Response(item_url, status=200, body=b'data')}
+            meta={'response': Response(item_url, status=200, body=output_img.getvalue())}
         )
 
     def inc_stats(self, *args, **kwargs):
@@ -379,9 +383,13 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
         self._mockcalled.append('file_path')
         return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info)
 
+    def thumb_path(self, request, thumb_id, response=None, info=None):
+        self._mockcalled.append('thumb_path')
+        return super(MockedMediaPipelineDeprecatedMethods, self).thumb_path(request, thumb_id, response, info)
+
     def get_images(self, response, request, info):
         self._mockcalled.append('get_images')
-        return []
+        return super(MockedMediaPipelineDeprecatedMethods, self).get_images(response, request, info)
 
     def image_downloaded(self, response, request, info):
         self._mockcalled.append('image_downloaded')
@@ -392,7 +400,11 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
     skip = skip_pillow
 
     def setUp(self):
-        self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func)
+        self.pipe = MockedMediaPipelineDeprecatedMethods(
+            store_uri='store-uri',
+            download_func=_mocked_download_func,
+            settings=Settings({"IMAGES_THUMBS": {'small': (50, 50)}})
+        )
         self.pipe.open_spider(None)
         self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])
 
@@ -444,6 +456,16 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
         )
         self._assert_method_called_with_warnings('file_path', message, warnings)
 
+    @inlineCallbacks
+    def test_thumb_path_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            'thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, '
+            'please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)'
+        )
+        self._assert_method_called_with_warnings('thumb_path', message, warnings)
+
     @inlineCallbacks
     def test_get_images_called(self):
         yield self.pipe.process_item(self.item, None)

From 896f16f2def7c276350421352dddf6e7b0145519 Mon Sep 17 00:00:00 2001
From: Mohammadtaher Abbasi <mohammadtaher.abbasi@zoodfood.com>
Date: Wed, 25 May 2022 23:59:25 +0430
Subject: [PATCH 598/716] make thumb_path method backwards compatible

---
 scrapy/pipelines/images.py | 2 +-
 scrapy/pipelines/media.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 45ac03820..6b97190ee 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -179,6 +179,6 @@ class ImagesPipeline(FilesPipeline):
         image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return f'full/{image_guid}.jpg'
 
-    def thumb_path(self, request, thumb_id, response=None, info=None, item=None):
+    def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
         thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return f'thumbs/{thumb_id}/{thumb_guid}.jpg'
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index d1bccf323..430c37227 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -121,7 +121,7 @@ class MediaPipeline:
     def _make_compatible(self):
         """Make overridable methods of MediaPipeline and subclasses backwards compatible"""
         methods = [
-            "file_path", "media_to_download", "media_downloaded",
+            "file_path", "thumb_path", "media_to_download", "media_downloaded",
             "file_downloaded", "image_downloaded", "get_images"
         ]
 

From c5627af15bcf413c04539aeb47dd07cf8b3e4092 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 7 Jun 2022 18:44:54 +0200
Subject: [PATCH 599/716] Centralize request fingerprints (#4524)

Co-authored-by: Mikhail Korobov <kmike84@gmail.com>
---
 docs/conf.py                                  |   2 +
 docs/news.rst                                 |   2 +-
 docs/topics/api.rst                           |   7 +
 docs/topics/item-pipeline.rst                 |   4 +-
 docs/topics/request-response.rst              | 268 +++++++
 docs/topics/settings.rst                      |   8 +-
 scrapy/crawler.py                             |   7 +
 scrapy/dupefilters.py                         |  49 +-
 scrapy/extensions/httpcache.py                |  14 +-
 scrapy/pipelines/media.py                     |   4 +-
 scrapy/settings/default_settings.py           |   3 +
 .../templates/project/module/settings.py.tmpl |   3 +
 scrapy/utils/request.py                       | 235 +++++-
 scrapy/utils/test.py                          |   9 +-
 tests/test_crawler.py                         |   3 +-
 tests/test_dupefilters.py                     |  80 +-
 tests/test_pipeline_files.py                  |   5 +-
 tests/test_pipeline_media.py                  |  53 +-
 tests/test_utils_request.py                   | 699 ++++++++++++++++--
 19 files changed, 1304 insertions(+), 151 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 55aa72d5a..378b01804 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -294,7 +294,9 @@ intersphinx_mapping = {
     'tox': ('https://tox.readthedocs.io/en/latest', None),
     'twisted': ('https://twistedmatrix.com/documents/current', None),
     'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
+    'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
 }
+intersphinx_disabled_reftypes = []
 
 
 # Options for sphinx-hoverxref options
diff --git a/docs/news.rst b/docs/news.rst
index 5d92067b5..2d0ab485e 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1643,7 +1643,7 @@ New features
     :issue:`4370`)
 
 *   A new ``keep_fragments`` parameter of
-    :func:`scrapy.utils.request.request_fingerprint` allows to generate
+    ``scrapy.utils.request.request_fingerprint`` allows to generate
     different fingerprints for requests with different fragments in their URL
     (:issue:`4104`)
 
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 900b19c7a..60b5acd10 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares
     :class:`scrapy.Spider` subclass and a
     :class:`scrapy.settings.Settings` object.
 
+    .. attribute:: request_fingerprinter
+
+        The request fingerprint builder of this crawler.
+
+        This is used from extensions and middlewares to build short, unique
+        identifiers for requests. See :ref:`request-fingerprints`.
+
     .. attribute:: settings
 
         The settings manager of this crawler.
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 391751364..882ff5661 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -60,9 +60,9 @@ Additionally, they may also implement the following methods:
    :param spider: the spider which was closed
    :type spider: :class:`~scrapy.Spider` object
 
-.. method:: from_crawler(cls, crawler)
+.. classmethod:: from_crawler(cls, crawler)
 
-   If present, this classmethod is called to create a pipeline instance
+   If present, this class method is called to create a pipeline instance
    from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
    of the pipeline. Crawler object provides access to all Scrapy core
    components like settings and signals; it is a way for pipeline to
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 92a471faf..49cb69f67 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -339,6 +339,7 @@ errors if needed::
                 request = failure.request
                 self.logger.error('TimeoutError on %s', request.url)
 
+
 .. _errback-cb_kwargs:
 
 Accessing additional data in errback functions
@@ -364,6 +365,273 @@ achieve this by using ``Failure.request.cb_kwargs``::
             main_url=failure.request.cb_kwargs['main_url'],
         )
 
+
+.. _request-fingerprints:
+
+Request fingerprints
+--------------------
+
+There are some aspects of scraping, such as filtering out duplicate requests
+(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
+:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
+unique identifier from a :class:`~scrapy.http.Request` object: a request
+fingerprint.
+
+You often do not need to worry about request fingerprints, the default request
+fingerprinter works for most projects.
+
+However, there is no universal way to generate a unique identifier from a
+request, because different situations require comparing requests differently.
+For example, sometimes you may need to compare URLs case-insensitively, include
+URL fragments, exclude certain URL query parameters, include some or all
+headers, etc.
+
+To change how request fingerprints are built for your requests, use the
+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
+
+.. setting:: REQUEST_FINGERPRINTER_CLASS
+
+REQUEST_FINGERPRINTER_CLASS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: VERSION
+
+Default: :class:`scrapy.utils.request.RequestFingerprinter`
+
+A :ref:`request fingerprinter class <custom-request-fingerprinter>` or its
+import path.
+
+.. autoclass:: scrapy.utils.request.RequestFingerprinter
+
+
+.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
+
+REQUEST_FINGERPRINTER_IMPLEMENTATION
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: VERSION
+
+Default: ``'PREVIOUS_VERSION'``
+
+Determines which request fingerprinting algorithm is used by the default
+request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
+
+Possible values are:
+
+-   ``'PREVIOUS_VERSION'`` (default)
+
+    This implementation uses the same request fingerprinting algorithm as
+    Scrapy PREVIOUS_VERSION and earlier versions.
+
+    Even though this is the default value for backward compatibility reasons,
+    it is a deprecated value.
+
+-   ``'VERSION'``
+
+    This implementation was introduced in Scrapy VERSION to fix an issue of the
+    previous implementation.
+
+    New projects should use this value. The :command:`startproject` command
+    sets this value in the generated ``settings.py`` file.
+
+If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are
+using Scrapy components where changing the request fingerprinting algorithm
+would cause undesired results, you need to carefully decide when to change the
+value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
+setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request
+fingerprinting algorithm and does not log this warning (
+:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a
+class).
+
+Scenarios where changing the request fingerprinting algorithm may cause
+undesired results include, for example, using the HTTP cache middleware (see
+:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
+Changing the request fingerprinting algorithm would invalidade the current
+cache, requiring you to redownload all requests again.
+
+Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in
+your settings to switch already to the request fingerprinting implementation
+that will be the only request fingerprinting implementation available in a
+future version of Scrapy, and remove the deprecation warning triggered by using
+the default value (``'PREVIOUS_VERSION'``).
+
+
+.. _PREVIOUS_VERSION-request-fingerprinter:
+.. _custom-request-fingerprinter:
+
+Writing your own request fingerprinter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A request fingerprinter is a class that must implement the following method:
+
+.. method:: fingerprint(self, request)
+
+   Return a :class:`bytes` object that uniquely identifies *request*.
+
+   See also :ref:`request-fingerprint-restrictions`.
+
+   :param request: request to fingerprint
+   :type request: scrapy.http.Request
+
+Additionally, it may also implement the following methods:
+
+.. classmethod:: from_crawler(cls, crawler)
+
+   If present, this class method is called to create a request fingerprinter
+   instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
+   new instance of the request fingerprinter.
+
+   *crawler* provides access to all Scrapy core components like settings and
+   signals; it is a way for the request fingerprinter to access them and hook
+   its functionality into Scrapy.
+
+   :param crawler: crawler that uses this request fingerprinter
+   :type crawler: :class:`~scrapy.crawler.Crawler` object
+
+.. classmethod:: from_settings(cls, settings)
+
+   If present, and ``from_crawler`` is not defined, this class method is called
+   to create a request fingerprinter instance from a
+   :class:`~scrapy.settings.Settings` object. It must return a new instance of
+   the request fingerprinter.
+
+The ``fingerprint`` method of the default request fingerprinter,
+:class:`scrapy.utils.request.RequestFingerprinter`, uses
+:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
+common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well
+in your ``fingerprint`` method implementation:
+
+.. autofunction:: scrapy.utils.request.fingerprint
+
+For example, to take the value of a request header named ``X-ID`` into
+account::
+
+    # my_project/settings.py
+    REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
+
+    # my_project/utils.py
+    from scrapy.utils.request import fingerprint
+
+    class RequestFingerprinter:
+
+        def fingerprint(self, request):
+            return fingerprint(request, include_headers=['X-ID'])
+
+You can also write your own fingerprinting logic from scratch.
+
+However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure
+you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
+
+-   Caching saves CPU by ensuring that fingerprints are calculated only once
+    per request, and not once per Scrapy component that needs the fingerprint
+    of a request.
+
+-   Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
+    request objects do not stay in memory forever just because you have
+    references to them in your cache dictionary.
+
+For example, to take into account only the URL of a request, without any prior
+URL canonicalization or taking the request method or body into account::
+
+    from hashlib import sha1
+    from weakref import WeakKeyDictionary
+
+    from scrapy.utils.python import to_bytes
+
+    class RequestFingerprinter:
+
+        cache = WeakKeyDictionary()
+
+        def fingerprint(self, request):
+            if request not in self.cache:
+                fp = sha1()
+                fp.update(to_bytes(request.url))
+                self.cache[request] = fp.digest()
+            return self.cache[request]
+
+If you need to be able to override the request fingerprinting for arbitrary
+requests from your spider callbacks, you may implement a request fingerprinter
+that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
+when available, and then falls back to
+:func:`~scrapy.utils.request.fingerprint`. For example::
+
+    from scrapy.utils.request import fingerprint
+
+    class RequestFingerprinter:
+
+        def fingerprint(self, request):
+            if 'fingerprint' in request.meta:
+                return request.meta['fingerprint']
+            return fingerprint(request)
+
+If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION
+without using the deprecated ``'PREVIOUS_VERSION'`` value of the
+:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
+request fingerprinter::
+
+    from hashlib import sha1
+    from weakref import WeakKeyDictionary
+
+    from scrapy.utils.python import to_bytes
+    from w3lib.url import canonicalize_url
+
+    class RequestFingerprinter:
+
+        cache = WeakKeyDictionary()
+
+        def fingerprint(self, request):
+            if request not in self.cache:
+                fp = sha1()
+                fp.update(to_bytes(request.method))
+                fp.update(to_bytes(canonicalize_url(request.url)))
+                fp.update(request.body or b'')
+                self.cache[request] = fp.digest()
+            return self.cache[request]
+
+
+.. _request-fingerprint-restrictions:
+
+Request fingerprint restrictions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Scrapy components that use request fingerprints may impose additional
+restrictions on the format of the fingerprints that your :ref:`request
+fingerprinter <custom-request-fingerprinter>` generates.
+
+The following built-in Scrapy components have such restrictions:
+
+-   :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
+    value of :setting:`HTTPCACHE_STORAGE`)
+
+    Request fingerprints must be at least 1 byte long.
+
+    Path and filename length limits of the file system of
+    :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
+    the following directory structure is created:
+
+    -   :attr:`Spider.name <scrapy.spiders.Spider.name>`
+
+        -   first byte of a request fingerprint as hexadecimal
+
+            -   fingerprint as hexadecimal
+
+                -   filenames up to 16 characters long
+
+    For example, if a request fingerprint is made of 20 bytes (default),
+    :setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
+    and the name of your spider is ``'my_spider'`` your file system must
+    support a file path like::
+
+        /home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
+
+-   :class:`scrapy.extensions.httpcache.DbmCacheStorage`
+
+    The underlying DBM implementation must support keys as long as twice
+    the number of bytes of a request fingerprint, plus 5. For example,
+    if a request fingerprint is made of 20 bytes (default),
+    45-character-long keys must be supported.
+
+
 .. _topics-request-meta:
 
 Request.meta special keys
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4e105642d..2046c6446 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -825,12 +825,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
 
 The class used to detect and filter duplicate requests.
 
-The default (``RFPDupeFilter``) filters based on request fingerprint using
-the ``scrapy.utils.request.request_fingerprint`` function. In order to change
-the way duplicates are checked you could subclass ``RFPDupeFilter`` and
-override its ``request_fingerprint`` method. This method should accept
-scrapy :class:`~scrapy.Request` object and return its fingerprint
-(a string).
+The default (``RFPDupeFilter``) filters based on the
+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
 
 You can disable filtering of duplicate requests by setting
 :setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index a638254f1..fdca7b335 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -51,6 +51,7 @@ class Crawler:
         self.spidercls.update_settings(self.settings)
 
         self.signals = SignalManager(self)
+
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
 
         handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
@@ -71,6 +72,12 @@ class Crawler:
         lf_cls = load_object(self.settings['LOG_FORMATTER'])
         self.logformatter = lf_cls.from_crawler(self)
 
+        self.request_fingerprinter = create_instance(
+            load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
+            settings=self.settings,
+            crawler=self,
+        )
+
         reactor_class = self.settings.get("TWISTED_REACTOR")
         if init_reactor:
             # this needs to be done after the spider settings are merged,
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 292c68099..d1b0559ef 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,14 +1,16 @@
 import logging
 import os
 from typing import Optional, Set, Type, TypeVar
+from warnings import warn
 
 from twisted.internet.defer import Deferred
 
 from scrapy.http.request import Request
 from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.job import job_dir
-from scrapy.utils.request import referer_str, request_fingerprint
+from scrapy.utils.request import referer_str, RequestFingerprinter
 
 
 BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
@@ -39,8 +41,15 @@ RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
 class RFPDupeFilter(BaseDupeFilter):
     """Request Fingerprint duplicates filter"""
 
-    def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:
+    def __init__(
+        self,
+        path: Optional[str] = None,
+        debug: bool = False,
+        *,
+        fingerprinter=None,
+    ) -> None:
         self.file = None
+        self.fingerprinter = fingerprinter or RequestFingerprinter()
         self.fingerprints: Set[str] = set()
         self.logdupes = True
         self.debug = debug
@@ -51,9 +60,39 @@ class RFPDupeFilter(BaseDupeFilter):
             self.fingerprints.update(x.rstrip() for x in self.file)
 
     @classmethod
-    def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:
+    def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:
         debug = settings.getbool('DUPEFILTER_DEBUG')
-        return cls(job_dir(settings), debug)
+        try:
+            return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
+        except TypeError:
+            warn(
+                "RFPDupeFilter subclasses must either modify their '__init__' "
+                "method to support a 'fingerprinter' parameter or reimplement "
+                "the 'from_settings' class method.",
+                ScrapyDeprecationWarning,
+            )
+            result = cls(job_dir(settings), debug)
+            result.fingerprinter = fingerprinter
+            return result
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        try:
+            return cls.from_settings(
+                crawler.settings,
+                fingerprinter=crawler.request_fingerprinter,
+            )
+        except TypeError:
+            warn(
+                "RFPDupeFilter subclasses must either modify their overridden "
+                "'__init__' method and 'from_settings' class method to "
+                "support a 'fingerprinter' parameter, or reimplement the "
+                "'from_crawler' class method.",
+                ScrapyDeprecationWarning,
+            )
+            result = cls.from_settings(crawler.settings)
+            result.fingerprinter = crawler.request_fingerprinter
+            return result
 
     def request_seen(self, request: Request) -> bool:
         fp = self.request_fingerprint(request)
@@ -65,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter):
         return False
 
     def request_fingerprint(self, request: Request) -> str:
-        return request_fingerprint(request)
+        return self.fingerprinter.fingerprint(request).hex()
 
     def close(self, reason: str) -> None:
         if self.file:
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index d0ae29b90..c71484cfa 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypes
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.project import data_path
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.request import request_fingerprint
 
 
 logger = logging.getLogger(__name__)
@@ -228,6 +227,8 @@ class DbmCacheStorage:
 
         logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
 
+        self._fingerprinter = spider.crawler.request_fingerprinter
+
     def close_spider(self, spider):
         self.db.close()
 
@@ -244,7 +245,7 @@ class DbmCacheStorage:
         return response
 
     def store_response(self, spider, request, response):
-        key = self._request_key(request)
+        key = self._fingerprinter.fingerprint(request).hex()
         data = {
             'status': response.status,
             'url': response.url,
@@ -255,7 +256,7 @@ class DbmCacheStorage:
         self.db[f'{key}_time'] = str(time())
 
     def _read_data(self, spider, request):
-        key = self._request_key(request)
+        key = self._fingerprinter.fingerprint(request).hex()
         db = self.db
         tkey = f'{key}_time'
         if tkey not in db:
@@ -267,9 +268,6 @@ class DbmCacheStorage:
 
         return pickle.loads(db[f'{key}_data'])
 
-    def _request_key(self, request):
-        return request_fingerprint(request)
-
 
 class FilesystemCacheStorage:
 
@@ -283,6 +281,8 @@ class FilesystemCacheStorage:
         logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
                      extra={'spider': spider})
 
+        self._fingerprinter = spider.crawler.request_fingerprinter
+
     def close_spider(self, spider):
         pass
 
@@ -329,7 +329,7 @@ class FilesystemCacheStorage:
             f.write(request.body)
 
     def _get_request_path(self, spider, request):
-        key = request_fingerprint(request)
+        key = self._fingerprinter.fingerprint(request).hex()
         return os.path.join(self.cachedir, spider.name, key[0:2], key)
 
     def _read_meta(self, spider, request):
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 430c37227..5308a9793 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -11,7 +11,6 @@ from scrapy.settings import Settings
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
-from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.log import failure_to_exc_info
 
@@ -77,6 +76,7 @@ class MediaPipeline:
         except AttributeError:
             pipe = cls()
         pipe.crawler = crawler
+        pipe._fingerprinter = crawler.request_fingerprinter
         return pipe
 
     def open_spider(self, spider):
@@ -90,7 +90,7 @@ class MediaPipeline:
         return dfd.addCallback(self.item_completed, item, info)
 
     def _process_request(self, request, info, item):
-        fp = request_fingerprint(request)
+        fp = self._fingerprinter.fingerprint(request)
         cb = request.callback or (lambda _: _)
         eb = request.errback
         request.callback = None
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8389a70cb..f5a3efe69 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -246,6 +246,9 @@ REDIRECT_PRIORITY_ADJUST = +2
 REFERER_ENABLED = True
 REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
 
+REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
+REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION'
+
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
 RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index a414b5fde..5e541e2c0 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -86,3 +86,6 @@ ROBOTSTXT_OBEY = True
 #HTTPCACHE_DIR = 'httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION'
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 70ef3ba2b..cf33317ce 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -4,7 +4,9 @@ scrapy.http.Request objects
 """
 
 import hashlib
-from typing import Dict, Iterable, Optional, Tuple, Union
+import json
+import warnings
+from typing import Dict, Iterable, List, Optional, Tuple, Union
 from urllib.parse import urlunparse
 from weakref import WeakKeyDictionary
 
@@ -12,13 +14,22 @@ from w3lib.http import basic_auth_header
 from w3lib.url import canonicalize_url
 
 from scrapy import Request, Spider
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode
 
 
-_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
-_fingerprint_cache = WeakKeyDictionary()
+_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
+_deprecated_fingerprint_cache = WeakKeyDictionary()
+
+
+def _serialize_headers(headers, request):
+    for header in headers:
+        if header in request.headers:
+            yield header
+            for value in request.headers.getlist(header):
+                yield value
 
 
 def request_fingerprint(
@@ -26,6 +37,123 @@ def request_fingerprint(
     include_headers: Optional[Iterable[Union[bytes, str]]] = None,
     keep_fragments: bool = False,
 ) -> str:
+    """
+    Return the request fingerprint as an hexadecimal string.
+
+    The request fingerprint is a hash that uniquely identifies the resource the
+    request points to. For example, take the following two urls:
+
+    http://www.example.com/query?id=111&cat=222
+    http://www.example.com/query?cat=222&id=111
+
+    Even though those are two different URLs both point to the same resource
+    and are equivalent (i.e. they should return the same response).
+
+    Another example are cookies used to store session ids. Suppose the
+    following page is only accessible to authenticated users:
+
+    http://www.example.com/members/offers.html
+
+    Lots of sites use a cookie to store the session id, which adds a random
+    component to the HTTP Request and thus should be ignored when calculating
+    the fingerprint.
+
+    For this reason, request headers are ignored by default when calculating
+    the fingerprint. If you want to include specific headers use the
+    include_headers argument, which is a list of Request headers to include.
+
+    Also, servers usually ignore fragments in urls when handling requests,
+    so they are also ignored by default when calculating the fingerprint.
+    If you want to include them, set the keep_fragments argument to True
+    (for instance when handling requests with a headless browser).
+    """
+    if include_headers or keep_fragments:
+        message = (
+            'Call to deprecated function '
+            'scrapy.utils.request.request_fingerprint().\n'
+            '\n'
+            'If you are using this function in a Scrapy component because you '
+            'need a non-default fingerprinting algorithm, and you are OK '
+            'with that non-default fingerprinting algorithm being used by '
+            'all Scrapy components and not just the one calling this '
+            'function, use crawler.request_fingerprinter.fingerprint() '
+            'instead in your Scrapy component (you can get the crawler '
+            'object from the \'from_crawler\' class method), and use the '
+            '\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your '
+            'non-default fingerprinting algorithm.\n'
+            '\n'
+            'Otherwise, consider using the '
+            'scrapy.utils.request.fingerprint() function instead.\n'
+            '\n'
+            'If you switch to \'fingerprint()\', or assign the '
+            '\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses '
+            '\'fingerprint()\', the generated fingerprints will not only be '
+            'bytes instead of a string, but they will also be different from '
+            'those generated by \'request_fingerprint()\'. Before you switch, '
+            'make sure that you understand the consequences of this (e.g. '
+            'cache invalidation) and are OK with them; otherwise, consider '
+            'implementing your own function which returns the same '
+            'fingerprints as the deprecated \'request_fingerprint()\' function.'
+        )
+    else:
+        message = (
+            'Call to deprecated function '
+            'scrapy.utils.request.request_fingerprint().\n'
+            '\n'
+            'If you are using this function in a Scrapy component, and you '
+            'are OK with users of your component changing the fingerprinting '
+            'algorithm through settings, use '
+            'crawler.request_fingerprinter.fingerprint() instead in your '
+            'Scrapy component (you can get the crawler object from the '
+            '\'from_crawler\' class method).\n'
+            '\n'
+            'Otherwise, consider using the '
+            'scrapy.utils.request.fingerprint() function instead.\n'
+            '\n'
+            'Either way, the resulting fingerprints will be returned as '
+            'bytes, not as a string, and they will also be different from '
+            'those generated by \'request_fingerprint()\'. Before you switch, '
+            'make sure that you understand the consequences of this (e.g. '
+            'cache invalidation) and are OK with them; otherwise, consider '
+            'implementing your own function which returns the same '
+            'fingerprints as the deprecated \'request_fingerprint()\' function.'
+        )
+    warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
+    processed_include_headers: Optional[Tuple[bytes, ...]] = None
+    if include_headers:
+        processed_include_headers = tuple(
+            to_bytes(h.lower()) for h in sorted(include_headers)
+        )
+    cache = _deprecated_fingerprint_cache.setdefault(request, {})
+    cache_key = (processed_include_headers, keep_fragments)
+    if cache_key not in cache:
+        fp = hashlib.sha1()
+        fp.update(to_bytes(request.method))
+        fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
+        fp.update(request.body or b'')
+        if processed_include_headers:
+            for part in _serialize_headers(processed_include_headers, request):
+                fp.update(part)
+        cache[cache_key] = fp.hexdigest()
+    return cache[cache_key]
+
+
+def _request_fingerprint_as_bytes(*args, **kwargs):
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        return bytes.fromhex(request_fingerprint(*args, **kwargs))
+
+
+_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
+_fingerprint_cache = WeakKeyDictionary()
+
+
+def fingerprint(
+    request: Request,
+    *,
+    include_headers: Optional[Iterable[Union[bytes, str]]] = None,
+    keep_fragments: bool = False,
+) -> bytes:
     """
     Return the request fingerprint.
 
@@ -43,7 +171,7 @@ def request_fingerprint(
 
     http://www.example.com/members/offers.html
 
-    Lot of sites use a cookie to store the session id, which adds a random
+    Lots of sites use a cookie to store the session id, which adds a random
     component to the HTTP Request and thus should be ignored when calculating
     the fingerprint.
 
@@ -55,29 +183,96 @@ def request_fingerprint(
     so they are also ignored by default when calculating the fingerprint.
     If you want to include them, set the keep_fragments argument to True
     (for instance when handling requests with a headless browser).
-
     """
-    headers: Optional[Tuple[bytes, ...]] = None
+    processed_include_headers: Optional[Tuple[bytes, ...]] = None
     if include_headers:
-        headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
+        processed_include_headers = tuple(
+            to_bytes(h.lower()) for h in sorted(include_headers)
+        )
     cache = _fingerprint_cache.setdefault(request, {})
-    cache_key = (headers, keep_fragments)
+    cache_key = (processed_include_headers, keep_fragments)
     if cache_key not in cache:
-        fp = hashlib.sha1()
-        fp.update(to_bytes(request.method))
-        fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
-        fp.update(request.body or b'')
-        if headers:
-            for hdr in headers:
-                if hdr in request.headers:
-                    fp.update(hdr)
-                    for v in request.headers.getlist(hdr):
-                        fp.update(v)
-        cache[cache_key] = fp.hexdigest()
+        # To decode bytes reliably (JSON does not support bytes), regardless of
+        # character encoding, we use bytes.hex()
+        headers: Dict[str, List[str]] = {}
+        if processed_include_headers:
+            for header in processed_include_headers:
+                if header in request.headers:
+                    headers[header.hex()] = [
+                        header_value.hex()
+                        for header_value in request.headers.getlist(header)
+                    ]
+        fingerprint_data = {
+            'method': to_unicode(request.method),
+            'url': canonicalize_url(request.url, keep_fragments=keep_fragments),
+            'body': (request.body or b'').hex(),
+            'headers': headers,
+        }
+        fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
+        cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()
     return cache[cache_key]
 
 
-def request_authenticate(request: Request, username: str, password: str) -> None:
+class RequestFingerprinter:
+    """Default fingerprinter.
+
+    It takes into account a canonical version
+    (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
+    <scrapy.http.Request.url>` and the values of :attr:`request.method
+    <scrapy.http.Request.method>` and :attr:`request.body
+    <scrapy.http.Request.body>`. It then generates an `SHA1
+    <https://en.wikipedia.org/wiki/SHA-1>`_ hash.
+
+    .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`.
+    """
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    def __init__(self, crawler=None):
+        if crawler:
+            implementation = crawler.settings.get(
+                'REQUEST_FINGERPRINTER_IMPLEMENTATION'
+            )
+        else:
+            implementation = 'PREVIOUS_VERSION'
+        if implementation == 'PREVIOUS_VERSION':
+            message = (
+                '\'PREVIOUS_VERSION\' is a deprecated value for the '
+                '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n'
+                '\n'
+                'It is also the default value. In other words, it is normal '
+                'to get this warning if you have not defined a value for the '
+                '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so '
+                'for backward compatibility reasons, but it will change in a '
+                'future version of Scrapy.\n'
+                '\n'
+                'See the documentation of the '
+                '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for '
+                'information on how to handle this deprecation.'
+            )
+            warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
+            self._fingerprint = _request_fingerprint_as_bytes
+        elif implementation == 'VERSION':
+            self._fingerprint = fingerprint
+        else:
+            raise ValueError(
+                f'Got an invalid value on setting '
+                f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '
+                f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) '
+                f'and \'VERSION\'.'
+            )
+
+    def fingerprint(self, request):
+        return self._fingerprint(request)
+
+
+def request_authenticate(
+    request: Request,
+    username: str,
+    password: str,
+) -> None:
     """Authenticate the given request (in place) using the HTTP basic access
     authentication mechanism (RFC 2617) and the given username and password
     """
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 24c38283a..b90ea5009 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -54,7 +54,7 @@ def get_ftp_content_and_delete(
     return "".join(ftp_data)
 
 
-def get_crawler(spidercls=None, settings_dict=None):
+def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level
     priority.
@@ -62,7 +62,12 @@ def get_crawler(spidercls=None, settings_dict=None):
     from scrapy.crawler import CrawlerRunner
     from scrapy.spiders import Spider
 
-    runner = CrawlerRunner(settings_dict)
+    # Set by default settings that prevent deprecation warnings.
+    settings = {}
+    if prevent_warnings:
+        settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION'
+    settings.update(settings_dict or {})
+    runner = CrawlerRunner(settings)
     return runner.create_crawler(spidercls or Spider)
 
 
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8f6227109..f7aa769e4 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -104,7 +104,8 @@ class CrawlerLoggingTestCase(unittest.TestCase):
             custom_settings = {
                 'LOG_LEVEL': 'INFO',
                 'LOG_FILE': log_file,
-                # disable telnet if not available to avoid an extra warning
+                # settings to avoid extra warnings
+                'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
                 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
             }
 
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 680bb6dc8..b7df2554a 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -15,6 +15,16 @@ from scrapy.utils.test import get_crawler
 from tests.spiders import SimpleSpider
 
 
+def _get_dupefilter(*, crawler=None, settings=None, open=True):
+    if crawler is None:
+        crawler = get_crawler(settings_dict=settings)
+    scheduler = Scheduler.from_crawler(crawler)
+    dupefilter = scheduler.df
+    if open:
+        dupefilter.open()
+    return dupefilter
+
+
 class FromCrawlerRFPDupeFilter(RFPDupeFilter):
 
     @classmethod
@@ -64,9 +74,7 @@ class RFPDupeFilterTest(unittest.TestCase):
         self.assertEqual(scheduler.df.method, 'n/a')
 
     def test_filter(self):
-        dupefilter = RFPDupeFilter()
-        dupefilter.open()
-
+        dupefilter = _get_dupefilter()
         r1 = Request('http://scrapytest.org/1')
         r2 = Request('http://scrapytest.org/2')
         r3 = Request('http://scrapytest.org/2')
@@ -85,7 +93,7 @@ class RFPDupeFilterTest(unittest.TestCase):
 
         path = tempfile.mkdtemp()
         try:
-            df = RFPDupeFilter(path)
+            df = _get_dupefilter(settings={'JOBDIR': path}, open=False)
             try:
                 df.open()
                 assert not df.request_seen(r1)
@@ -93,7 +101,8 @@ class RFPDupeFilterTest(unittest.TestCase):
             finally:
                 df.close('finished')
 
-            df2 = RFPDupeFilter(path)
+            df2 = _get_dupefilter(settings={'JOBDIR': path}, open=False)
+            assert df != df2
             try:
                 df2.open()
                 assert df2.request_seen(r1)
@@ -109,26 +118,24 @@ class RFPDupeFilterTest(unittest.TestCase):
         output of request_seen.
 
         """
+        dupefilter = _get_dupefilter()
         r1 = Request('http://scrapytest.org/index.html')
         r2 = Request('http://scrapytest.org/INDEX.html')
 
-        dupefilter = RFPDupeFilter()
-        dupefilter.open()
-
         assert not dupefilter.request_seen(r1)
         assert not dupefilter.request_seen(r2)
 
         dupefilter.close('finished')
 
-        class CaseInsensitiveRFPDupeFilter(RFPDupeFilter):
+        class RequestFingerprinter:
 
-            def request_fingerprint(self, request):
+            def fingerprint(self, request):
                 fp = hashlib.sha1()
                 fp.update(to_bytes(request.url.lower()))
-                return fp.hexdigest()
+                return fp.digest()
 
-        case_insensitive_dupefilter = CaseInsensitiveRFPDupeFilter()
-        case_insensitive_dupefilter.open()
+        settings = {'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter}
+        case_insensitive_dupefilter = _get_dupefilter(settings=settings)
 
         assert not case_insensitive_dupefilter.request_seen(r1)
         assert case_insensitive_dupefilter.request_seen(r2)
@@ -142,8 +149,10 @@ class RFPDupeFilterTest(unittest.TestCase):
         r1 = Request('http://scrapytest.org/1')
 
         path = tempfile.mkdtemp()
+        crawler = get_crawler(settings_dict={'JOBDIR': path})
         try:
-            df = RFPDupeFilter(path)
+            scheduler = Scheduler.from_crawler(crawler)
+            df = scheduler.df
             df.open()
             df.request_seen(r1)
             df.close('finished')
@@ -164,11 +173,8 @@ class RFPDupeFilterTest(unittest.TestCase):
             settings = {'DUPEFILTER_DEBUG': False,
                         'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
-            scheduler = Scheduler.from_crawler(crawler)
             spider = SimpleSpider.from_crawler(crawler)
-
-            dupefilter = scheduler.df
-            dupefilter.open()
+            dupefilter = _get_dupefilter(crawler=crawler)
 
             r1 = Request('http://scrapytest.org/index.html')
             r2 = Request('http://scrapytest.org/index.html')
@@ -193,11 +199,41 @@ class RFPDupeFilterTest(unittest.TestCase):
             settings = {'DUPEFILTER_DEBUG': True,
                         'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
-            scheduler = Scheduler.from_crawler(crawler)
             spider = SimpleSpider.from_crawler(crawler)
-
-            dupefilter = scheduler.df
-            dupefilter.open()
+            dupefilter = _get_dupefilter(crawler=crawler)
+
+            r1 = Request('http://scrapytest.org/index.html')
+            r2 = Request('http://scrapytest.org/index.html',
+                         headers={'Referer': 'http://scrapytest.org/INDEX.html'})
+
+            dupefilter.log(r1, spider)
+            dupefilter.log(r2, spider)
+
+            assert crawler.stats.get_value('dupefilter/filtered') == 2
+            log.check_present(
+                (
+                    'scrapy.dupefilters',
+                    'DEBUG',
+                    'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
+                )
+            )
+            log.check_present(
+                (
+                    'scrapy.dupefilters',
+                    'DEBUG',
+                    'Filtered duplicate request: <GET http://scrapytest.org/index.html>'
+                    ' (referer: http://scrapytest.org/INDEX.html)'
+                )
+            )
+
+            dupefilter.close('finished')
+
+    def test_log_debug_default_dupefilter(self):
+        with LogCapture() as log:
+            settings = {'DUPEFILTER_DEBUG': True}
+            crawler = get_crawler(SimpleSpider, settings_dict=settings)
+            spider = SimpleSpider.from_crawler(crawler)
+            dupefilter = _get_dupefilter(crawler=crawler)
 
             r1 = Request('http://scrapytest.org/index.html')
             r2 = Request('http://scrapytest.org/index.html',
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 0ff2045ed..4228173ed 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -25,6 +25,7 @@ from scrapy.pipelines.files import (
 from scrapy.settings import Settings
 from scrapy.utils.test import (
     assert_gcs_environ,
+    get_crawler,
     get_ftp_content_and_delete,
     get_gcs_content_and_delete,
     skip_if_no_boto,
@@ -47,7 +48,9 @@ class FilesPipelineTestCase(unittest.TestCase):
 
     def setUp(self):
         self.tempdir = mkdtemp()
-        self.pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir}))
+        settings_dict = {'FILES_STORE': self.tempdir}
+        crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
+        self.pipeline = FilesPipeline.from_crawler(crawler)
         self.pipeline.download_func = _mocked_download_func
         self.pipeline.open_spider(None)
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index a802c7cf1..84e867660 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -7,17 +7,17 @@ from twisted.python.failure import Failure
 from twisted.internet import reactor
 from twisted.internet.defer import Deferred, inlineCallbacks
 
+from scrapy import signals
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
-from scrapy.utils.request import request_fingerprint
+from scrapy.pipelines.files import FileException
 from scrapy.pipelines.images import ImagesPipeline
 from scrapy.pipelines.media import MediaPipeline
-from scrapy.pipelines.files import FileException
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.signal import disconnect_all
-from scrapy import signals
+from scrapy.utils.test import get_crawler
 
 
 try:
@@ -39,11 +39,14 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
     settings = None
 
     def setUp(self):
-        self.spider = Spider('media.com')
-        self.pipe = self.pipeline_class(download_func=_mocked_download_func,
-                                        settings=Settings(self.settings))
+        spider_cls = Spider
+        self.spider = spider_cls('media.com')
+        crawler = get_crawler(spider_cls, self.settings)
+        self.pipe = self.pipeline_class.from_crawler(crawler)
+        self.pipe.download_func = _mocked_download_func
         self.pipe.open_spider(self.spider)
         self.info = self.pipe.spiderinfo
+        self.fingerprint = crawler.request_fingerprinter.fingerprint
 
     def tearDown(self):
         for name, signal in vars(signals).items():
@@ -156,7 +159,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         self.assertEqual(failure.value.__context__, def_gen_return_exc)
 
         # Let's calculate the request fingerprint and fake some runtime data...
-        fp = request_fingerprint(request)
+        fp = self.fingerprint(request)
         info = self.pipe.spiderinfo
         info.downloading.add(fp)
         info.waiting[fp] = []
@@ -273,7 +276,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = dict(requests=req)  # pass a single item
         new_item = yield self.pipe.process_item(item, self.spider)
         assert new_item is item
-        assert request_fingerprint(req) in self.info.downloaded
+        self.assertIn(self.fingerprint(req), self.info.downloaded)
 
         # returns iterable of Requests
         req1 = Request('http://url1')
@@ -281,8 +284,8 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = dict(requests=iter([req1, req2]))
         new_item = yield self.pipe.process_item(item, self.spider)
         assert new_item is item
-        assert request_fingerprint(req1) in self.info.downloaded
-        assert request_fingerprint(req2) in self.info.downloaded
+        assert self.fingerprint(req1) in self.info.downloaded
+        assert self.fingerprint(req2) in self.info.downloaded
 
     @inlineCallbacks
     def test_results_are_cached_across_multiple_items(self):
@@ -298,7 +301,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = dict(requests=req2)
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertTrue(new_item is item)
-        self.assertEqual(request_fingerprint(req1), request_fingerprint(req2))
+        self.assertEqual(self.fingerprint(req1), self.fingerprint(req2))
         self.assertEqual(new_item['results'], [(True, rsp1)])
 
     @inlineCallbacks
@@ -314,7 +317,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
     @inlineCallbacks
     def test_wait_if_request_is_downloading(self):
         def _check_downloading(response):
-            fp = request_fingerprint(req1)
+            fp = self.fingerprint(req1)
             self.assertTrue(fp in self.info.downloading)
             self.assertTrue(fp in self.info.waiting)
             self.assertTrue(fp not in self.info.downloaded)
@@ -351,7 +354,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
 class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
 
     def __init__(self, *args, **kwargs):
-        super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._mockcalled = []
 
     def get_media_requests(self, item, info):
@@ -369,19 +372,19 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
 
     def media_to_download(self, request, info):
         self._mockcalled.append('media_to_download')
-        return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info)
+        return super().media_to_download(request, info)
 
     def media_downloaded(self, response, request, info):
         self._mockcalled.append('media_downloaded')
-        return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info)
+        return super().media_downloaded(response, request, info)
 
     def file_downloaded(self, response, request, info):
         self._mockcalled.append('file_downloaded')
-        return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info)
+        return super().file_downloaded(response, request, info)
 
     def file_path(self, request, response=None, info=None):
         self._mockcalled.append('file_path')
-        return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info)
+        return super().file_path(request, response, info)
 
     def thumb_path(self, request, thumb_id, response=None, info=None):
         self._mockcalled.append('thumb_path')
@@ -393,18 +396,20 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
 
     def image_downloaded(self, response, request, info):
         self._mockcalled.append('image_downloaded')
-        return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info)
+        return super().image_downloaded(response, request, info)
 
 
 class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
     skip = skip_pillow
 
     def setUp(self):
-        self.pipe = MockedMediaPipelineDeprecatedMethods(
-            store_uri='store-uri',
-            download_func=_mocked_download_func,
-            settings=Settings({"IMAGES_THUMBS": {'small': (50, 50)}})
-        )
+        settings_dict = {
+            'IMAGES_STORE': 'store-uri',
+            'IMAGES_THUMBS': {'small': (50, 50)},
+        }
+        crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
+        self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)
+        self.pipe.download_func = _mocked_download_func
         self.pipe.open_spider(None)
         self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])
 
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 7e0049b1d..e9edfee98 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,73 +1,29 @@
 import unittest
+import warnings
+from hashlib import sha1
+from typing import Dict, Mapping, Optional, Tuple, Union
+from weakref import WeakKeyDictionary
+
+import pytest
+from w3lib.url import canonicalize_url
+
 from scrapy.http import Request
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.python import to_bytes
 from scrapy.utils.request import (
+    _deprecated_fingerprint_cache,
     _fingerprint_cache,
+    _request_fingerprint_as_bytes,
+    fingerprint,
     request_authenticate,
     request_fingerprint,
     request_httprepr,
 )
+from scrapy.utils.test import get_crawler
 
 
 class UtilsRequestTest(unittest.TestCase):
 
-    def test_request_fingerprint(self):
-        r1 = Request("http://www.example.com/query?id=111&cat=222")
-        r2 = Request("http://www.example.com/query?cat=222&id=111")
-        self.assertEqual(request_fingerprint(r1), request_fingerprint(r1))
-        self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
-
-        r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
-        r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
-        self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
-
-        # make sure caching is working
-        self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][(None, False)])
-
-        r1 = Request("http://www.example.com/members/offers.html")
-        r2 = Request("http://www.example.com/members/offers.html")
-        r2.headers['SESSIONID'] = b"somehash"
-        self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
-
-        r1 = Request("http://www.example.com/")
-        r2 = Request("http://www.example.com/")
-        r2.headers['Accept-Language'] = b'en'
-        r3 = Request("http://www.example.com/")
-        r3.headers['Accept-Language'] = b'en'
-        r3.headers['SESSIONID'] = b"somehash"
-
-        self.assertEqual(request_fingerprint(r1), request_fingerprint(r2), request_fingerprint(r3))
-
-        self.assertEqual(request_fingerprint(r1),
-                         request_fingerprint(r1, include_headers=['Accept-Language']))
-
-        self.assertNotEqual(
-            request_fingerprint(r1),
-            request_fingerprint(r2, include_headers=['Accept-Language']))
-
-        self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
-                         request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
-
-        r1 = Request("http://www.example.com/test.html")
-        r2 = Request("http://www.example.com/test.html#fragment")
-        self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
-        self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, keep_fragments=True))
-        self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r2, keep_fragments=True))
-        self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2, keep_fragments=True))
-
-        r1 = Request("http://www.example.com")
-        r2 = Request("http://www.example.com", method='POST')
-        r3 = Request("http://www.example.com", method='POST', body=b'request body')
-
-        self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
-        self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r3))
-
-        # cached fingerprint must be cleared on request copy
-        r1 = Request("http://www.example.com")
-        fp1 = request_fingerprint(r1)
-        r2 = r1.replace(url="http://www.example.com/other")
-        fp2 = request_fingerprint(r2)
-        self.assertNotEqual(fp1, fp2)
-
     def test_request_authenticate(self):
         r = Request("http://www.example.com")
         request_authenticate(r, 'someuser', 'somepass')
@@ -93,5 +49,632 @@ class UtilsRequestTest(unittest.TestCase):
         request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
 
 
+class FingerprintTest(unittest.TestCase):
+    maxDiff = None
+
+    function = staticmethod(fingerprint)
+    cache: Union[
+        "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]",
+        "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]",
+    ] = _fingerprint_cache
+    default_cache_key = (None, False)
+    known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
+        (
+            Request("http://example.org"),
+            b'xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD',
+            {},
+        ),
+        (
+            Request("https://example.org"),
+            b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
+            {},
+        ),
+        (
+            Request("https://example.org?a"),
+            b'G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c',
+            {},
+        ),
+        (
+            Request("https://example.org?a=b"),
+            b'\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17',
+            {},
+        ),
+        (
+            Request("https://example.org?a=b&a"),
+            b't+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96',
+            {},
+        ),
+        (
+            Request("https://example.org?a=b&a=c"),
+            b'\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef',
+            {},
+        ),
+        (
+            Request("https://example.org", method='POST'),
+            b'\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d',
+            {},
+        ),
+        (
+            Request("https://example.org", body=b'a'),
+            b'\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_',
+            {},
+        ),
+        (
+            Request("https://example.org", method='POST', body=b'a'),
+            b'5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7',
+            {},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
+            {},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            b']\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.',
+            {'include_headers': ['A']},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            b'<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f',
+            {'keep_fragments': True},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            b'\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11',
+            {'include_headers': ['A'], 'keep_fragments': True},
+        ),
+        (
+            Request("https://example.org/ab"),
+            b'N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6',
+            {},
+        ),
+        (
+            Request("https://example.org/a", body=b'b'),
+            b'_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5',
+            {},
+        ),
+    )
+
+    def test_query_string_key_order(self):
+        r1 = Request("http://www.example.com/query?id=111&cat=222")
+        r2 = Request("http://www.example.com/query?cat=222&id=111")
+        self.assertEqual(self.function(r1), self.function(r1))
+        self.assertEqual(self.function(r1), self.function(r2))
+
+    def test_query_string_key_without_value(self):
+        r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
+        r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+        self.assertNotEqual(self.function(r1), self.function(r2))
+
+    def test_caching(self):
+        r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+        self.assertEqual(
+            self.function(r1),
+            self.cache[r1][self.default_cache_key]
+        )
+
+    def test_header(self):
+        r1 = Request("http://www.example.com/members/offers.html")
+        r2 = Request("http://www.example.com/members/offers.html")
+        r2.headers['SESSIONID'] = b"somehash"
+        self.assertEqual(self.function(r1), self.function(r2))
+
+    def test_headers(self):
+        r1 = Request("http://www.example.com/")
+        r2 = Request("http://www.example.com/")
+        r2.headers['Accept-Language'] = b'en'
+        r3 = Request("http://www.example.com/")
+        r3.headers['Accept-Language'] = b'en'
+        r3.headers['SESSIONID'] = b"somehash"
+
+        self.assertEqual(self.function(r1), self.function(r2), self.function(r3))
+
+        self.assertEqual(self.function(r1),
+                         self.function(r1, include_headers=['Accept-Language']))
+
+        self.assertNotEqual(
+            self.function(r1),
+            self.function(r2, include_headers=['Accept-Language']))
+
+        self.assertEqual(self.function(r3, include_headers=['accept-language', 'sessionid']),
+                         self.function(r3, include_headers=['SESSIONID', 'Accept-Language']))
+
+    def test_fragment(self):
+        r1 = Request("http://www.example.com/test.html")
+        r2 = Request("http://www.example.com/test.html#fragment")
+        self.assertEqual(self.function(r1), self.function(r2))
+        self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True))
+        self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True))
+        self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True))
+
+    def test_method_and_body(self):
+        r1 = Request("http://www.example.com")
+        r2 = Request("http://www.example.com", method='POST')
+        r3 = Request("http://www.example.com", method='POST', body=b'request body')
+
+        self.assertNotEqual(self.function(r1), self.function(r2))
+        self.assertNotEqual(self.function(r2), self.function(r3))
+
+    def test_request_replace(self):
+        # cached fingerprint must be cleared on request copy
+        r1 = Request("http://www.example.com")
+        fp1 = self.function(r1)
+        r2 = r1.replace(url="http://www.example.com/other")
+        fp2 = self.function(r2)
+        self.assertNotEqual(fp1, fp2)
+
+    def test_part_separation(self):
+        # An old implementation used to serialize request data in a way that
+        # would put the body right after the URL.
+        r1 = Request("http://www.example.com/foo")
+        fp1 = self.function(r1)
+        r2 = Request("http://www.example.com/f", body=b'oo')
+        fp2 = self.function(r2)
+        self.assertNotEqual(fp1, fp2)
+
+    def test_hashes(self):
+        """Test hardcoded hashes, to make sure future changes to not introduce
+        backward incompatibilities."""
+        actual = [
+            self.function(request, **kwargs)
+            for request, _, kwargs in self.known_hashes
+        ]
+        expected = [
+            _fingerprint
+            for _, _fingerprint, _ in self.known_hashes
+        ]
+        self.assertEqual(actual, expected)
+
+
+class RequestFingerprintTest(FingerprintTest):
+    function = staticmethod(request_fingerprint)
+    cache = _deprecated_fingerprint_cache
+    known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
+        (
+            Request("http://example.org"),
+            'b2e5245ef826fd9576c93bd6e392fce3133fab62',
+            {},
+        ),
+        (
+            Request("https://example.org"),
+            'bd10a0a89ea32cdee77917320f1309b0da87e892',
+            {},
+        ),
+        (
+            Request("https://example.org?a"),
+            '2fb7d48ae02f04b749f40caa969c0bc3c43204ce',
+            {},
+        ),
+        (
+            Request("https://example.org?a=b"),
+            '42e5fe149b147476e3f67ad0670c57b4cc57856a',
+            {},
+        ),
+        (
+            Request("https://example.org?a=b&a"),
+            'd23a9787cb56c6375c2cae4453c5a8c634526942',
+            {},
+        ),
+        (
+            Request("https://example.org?a=b&a=c"),
+            '9a18a7a8552a9182b7f1e05d33876409e421e5c5',
+            {},
+        ),
+        (
+            Request("https://example.org", method='POST'),
+            'ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6',
+            {},
+        ),
+        (
+            Request("https://example.org", body=b'a'),
+            '4bb136e54e715a4ea7a9dd1101831765d33f2d60',
+            {},
+        ),
+        (
+            Request("https://example.org", method='POST', body=b'a'),
+            '6c6595374a304b293be762f7b7be3f54e9947c65',
+            {},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            'bd10a0a89ea32cdee77917320f1309b0da87e892',
+            {},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            '515b633cb3ca502a33a9d8c890e889ec1e425e65',
+            {'include_headers': ['A']},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            '505c96e7da675920dfef58725e8c957dfdb38f47',
+            {'keep_fragments': True},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            'd6f673cdcb661b7970c2b9a00ee63e87d1e2e5da',
+            {'include_headers': ['A'], 'keep_fragments': True},
+        ),
+        (
+            Request("https://example.org/ab"),
+            '4e2870fee58582d6f81755e9b8fdefe3cba0c951',
+            {},
+        ),
+        (
+            Request("https://example.org/a", body=b'b'),
+            '4e2870fee58582d6f81755e9b8fdefe3cba0c951',
+            {},
+        ),
+    )
+
+    @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
+    def test_part_separation(self):
+        super().test_part_separation()
+
+    def test_deprecation_default_parameters(self):
+        with pytest.warns(ScrapyDeprecationWarning) as warnings:
+            self.function(Request("http://www.example.com"))
+        messages = [str(warning.message) for warning in warnings]
+        self.assertTrue(
+            any(
+                'Call to deprecated function' in message
+                for message in messages
+            )
+        )
+        self.assertFalse(any('non-default' in message for message in messages))
+
+    def test_deprecation_non_default_parameters(self):
+        with pytest.warns(ScrapyDeprecationWarning) as warnings:
+            self.function(Request("http://www.example.com"), keep_fragments=True)
+        messages = [str(warning.message) for warning in warnings]
+        self.assertTrue(
+            any(
+                'Call to deprecated function' in message
+                for message in messages
+            )
+        )
+        self.assertTrue(any('non-default' in message for message in messages))
+
+
+class RequestFingerprintAsBytesTest(FingerprintTest):
+    function = staticmethod(_request_fingerprint_as_bytes)
+    cache = _deprecated_fingerprint_cache
+    known_hashes = RequestFingerprintTest.known_hashes
+
+    def test_caching(self):
+        r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+        self.assertEqual(
+            self.function(r1),
+            bytes.fromhex(self.cache[r1][self.default_cache_key])
+        )
+
+    @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
+    def test_part_separation(self):
+        super().test_part_separation()
+
+    def test_hashes(self):
+        actual = [
+            self.function(request, **kwargs)
+            for request, _, kwargs in self.known_hashes
+        ]
+        expected = [
+            bytes.fromhex(_fingerprint)
+            for _, _fingerprint, _ in self.known_hashes
+        ]
+        self.assertEqual(actual, expected)
+
+
+_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary()
+
+
+def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False):
+    if include_headers:
+        include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
+    cache = _fingerprint_cache_2_6.setdefault(request, {})
+    cache_key = (include_headers, keep_fragments)
+    if cache_key not in cache:
+        fp = sha1()
+        fp.update(to_bytes(request.method))
+        fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
+        fp.update(request.body or b'')
+        if include_headers:
+            for hdr in include_headers:
+                if hdr in request.headers:
+                    fp.update(hdr)
+                    for v in request.headers.getlist(hdr):
+                        fp.update(v)
+        cache[cache_key] = fp.hexdigest()
+    return cache[cache_key]
+
+
+REQUEST_OBJECTS_TO_TEST = (
+    Request("http://www.example.com/"),
+    Request("http://www.example.com/query?id=111&cat=222"),
+    Request("http://www.example.com/query?cat=222&id=111"),
+    Request('http://www.example.com/hnnoticiaj1.aspx?78132,199'),
+    Request('http://www.example.com/hnnoticiaj1.aspx?78160,199'),
+    Request("http://www.example.com/members/offers.html"),
+    Request(
+        "http://www.example.com/members/offers.html",
+        headers={'SESSIONID': b"somehash"},
+    ),
+    Request(
+        "http://www.example.com/",
+        headers={'Accept-Language': b"en"},
+    ),
+    Request(
+        "http://www.example.com/",
+        headers={
+            'Accept-Language': b"en",
+            'SESSIONID': b"somehash",
+        },
+    ),
+    Request("http://www.example.com/test.html"),
+    Request("http://www.example.com/test.html#fragment"),
+    Request("http://www.example.com", method='POST'),
+    Request("http://www.example.com", method='POST', body=b'request body'),
+)
+
+
+class BackwardCompatibilityTestCase(unittest.TestCase):
+
+    def test_function_backward_compatibility(self):
+        include_headers_to_test = (
+            None,
+            ['Accept-Language'],
+            ['accept-language', 'sessionid'],
+            ['SESSIONID', 'Accept-Language'],
+        )
+        for request_object in REQUEST_OBJECTS_TO_TEST:
+            for include_headers in include_headers_to_test:
+                for keep_fragments in (False, True):
+                    with warnings.catch_warnings():
+                        warnings.simplefilter("ignore")
+                        fp = request_fingerprint(
+                            request_object,
+                            include_headers=include_headers,
+                            keep_fragments=keep_fragments,
+                        )
+                    old_fp = request_fingerprint_2_6(
+                        request_object,
+                        include_headers=include_headers,
+                        keep_fragments=keep_fragments,
+                    )
+                    self.assertEqual(fp, old_fp)
+
+    def test_component_backward_compatibility(self):
+        for request_object in REQUEST_OBJECTS_TO_TEST:
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                crawler = get_crawler(prevent_warnings=False)
+                fp = crawler.request_fingerprinter.fingerprint(request_object)
+            old_fp = request_fingerprint_2_6(request_object)
+            self.assertEqual(fp.hex(), old_fp)
+
+    def test_custom_component_backward_compatibility(self):
+        """Tests that the backward-compatible request fingerprinting class featured
+        in the documentation is indeed backward compatible and does not cause a
+        warning to be logged."""
+
+        class RequestFingerprinter:
+
+            cache = WeakKeyDictionary()
+
+            def fingerprint(self, request):
+                if request not in self.cache:
+                    fp = sha1()
+                    fp.update(to_bytes(request.method))
+                    fp.update(to_bytes(canonicalize_url(request.url)))
+                    fp.update(request.body or b'')
+                    self.cache[request] = fp.digest()
+                return self.cache[request]
+
+        for request_object in REQUEST_OBJECTS_TO_TEST:
+            with warnings.catch_warnings() as logged_warnings:
+                settings = {
+                    'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+                }
+                crawler = get_crawler(settings_dict=settings)
+                fp = crawler.request_fingerprinter.fingerprint(request_object)
+            old_fp = request_fingerprint_2_6(request_object)
+            self.assertEqual(fp.hex(), old_fp)
+            self.assertFalse(logged_warnings)
+
+
+class RequestFingerprinterTestCase(unittest.TestCase):
+
+    def test_default_implementation(self):
+        with warnings.catch_warnings(record=True) as logged_warnings:
+            crawler = get_crawler(prevent_warnings=False)
+        request = Request('https://example.com')
+        self.assertEqual(
+            crawler.request_fingerprinter.fingerprint(request),
+            _request_fingerprint_as_bytes(request),
+        )
+        self.assertTrue(logged_warnings)
+
+    def test_deprecated_implementation(self):
+        settings = {
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION',
+        }
+        with warnings.catch_warnings(record=True) as logged_warnings:
+            crawler = get_crawler(settings_dict=settings)
+        request = Request('https://example.com')
+        self.assertEqual(
+            crawler.request_fingerprinter.fingerprint(request),
+            _request_fingerprint_as_bytes(request),
+        )
+        self.assertTrue(logged_warnings)
+
+    def test_recommended_implementation(self):
+        settings = {
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
+        }
+        with warnings.catch_warnings(record=True) as logged_warnings:
+            crawler = get_crawler(settings_dict=settings)
+        request = Request('https://example.com')
+        self.assertEqual(
+            crawler.request_fingerprinter.fingerprint(request),
+            fingerprint(request),
+        )
+        self.assertFalse(logged_warnings)
+
+    def test_unknown_implementation(self):
+        settings = {
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.5',
+        }
+        with self.assertRaises(ValueError):
+            get_crawler(settings_dict=settings)
+
+
+class CustomRequestFingerprinterTestCase(unittest.TestCase):
+
+    def test_include_headers(self):
+
+        class RequestFingerprinter:
+
+            def fingerprint(self, request):
+                return fingerprint(request, include_headers=['X-ID'])
+
+        settings = {
+            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+        }
+        crawler = get_crawler(settings_dict=settings)
+
+        r1 = Request("http://www.example.com", headers={'X-ID': '1'})
+        fp1 = crawler.request_fingerprinter.fingerprint(r1)
+        r2 = Request("http://www.example.com", headers={'X-ID': '2'})
+        fp2 = crawler.request_fingerprinter.fingerprint(r2)
+        self.assertNotEqual(fp1, fp2)
+
+    def test_dont_canonicalize(self):
+
+        class RequestFingerprinter:
+            cache = WeakKeyDictionary()
+
+            def fingerprint(self, request):
+                if request not in self.cache:
+                    fp = sha1()
+                    fp.update(to_bytes(request.url))
+                    self.cache[request] = fp.digest()
+                return self.cache[request]
+
+        settings = {
+            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+        }
+        crawler = get_crawler(settings_dict=settings)
+
+        r1 = Request("http://www.example.com?a=1&a=2")
+        fp1 = crawler.request_fingerprinter.fingerprint(r1)
+        r2 = Request("http://www.example.com?a=2&a=1")
+        fp2 = crawler.request_fingerprinter.fingerprint(r2)
+        self.assertNotEqual(fp1, fp2)
+
+    def test_meta(self):
+
+        class RequestFingerprinter:
+
+            def fingerprint(self, request):
+                if 'fingerprint' in request.meta:
+                    return request.meta['fingerprint']
+                return fingerprint(request)
+
+        settings = {
+            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+        }
+        crawler = get_crawler(settings_dict=settings)
+
+        r1 = Request("http://www.example.com")
+        fp1 = crawler.request_fingerprinter.fingerprint(r1)
+        r2 = Request("http://www.example.com", meta={'fingerprint': 'a'})
+        fp2 = crawler.request_fingerprinter.fingerprint(r2)
+        r3 = Request("http://www.example.com", meta={'fingerprint': 'a'})
+        fp3 = crawler.request_fingerprinter.fingerprint(r3)
+        r4 = Request("http://www.example.com", meta={'fingerprint': 'b'})
+        fp4 = crawler.request_fingerprinter.fingerprint(r4)
+        self.assertNotEqual(fp1, fp2)
+        self.assertNotEqual(fp1, fp4)
+        self.assertNotEqual(fp2, fp4)
+        self.assertEqual(fp2, fp3)
+
+    def test_from_crawler(self):
+
+        class RequestFingerprinter:
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                return cls(crawler)
+
+            def __init__(self, crawler):
+                self._fingerprint = crawler.settings['FINGERPRINT']
+
+            def fingerprint(self, request):
+                return self._fingerprint
+
+        settings = {
+            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+            'FINGERPRINT': b'fingerprint',
+        }
+        crawler = get_crawler(settings_dict=settings)
+
+        request = Request("http://www.example.com")
+        fingerprint = crawler.request_fingerprinter.fingerprint(request)
+        self.assertEqual(fingerprint, settings['FINGERPRINT'])
+
+    def test_from_settings(self):
+
+        class RequestFingerprinter:
+
+            @classmethod
+            def from_settings(cls, settings):
+                return cls(settings)
+
+            def __init__(self, settings):
+                self._fingerprint = settings['FINGERPRINT']
+
+            def fingerprint(self, request):
+                return self._fingerprint
+
+        settings = {
+            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+            'FINGERPRINT': b'fingerprint',
+        }
+        crawler = get_crawler(settings_dict=settings)
+
+        request = Request("http://www.example.com")
+        fingerprint = crawler.request_fingerprinter.fingerprint(request)
+        self.assertEqual(fingerprint, settings['FINGERPRINT'])
+
+    def test_from_crawler_and_settings(self):
+
+        class RequestFingerprinter:
+
+            # This method is ignored due to the presence of from_crawler
+            @classmethod
+            def from_settings(cls, settings):
+                return cls(settings)
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                return cls(crawler)
+
+            def __init__(self, crawler):
+                self._fingerprint = crawler.settings['FINGERPRINT']
+
+            def fingerprint(self, request):
+                return self._fingerprint
+
+        settings = {
+            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+            'FINGERPRINT': b'fingerprint',
+        }
+        crawler = get_crawler(settings_dict=settings)
+
+        request = Request("http://www.example.com")
+        fingerprint = crawler.request_fingerprinter.fingerprint(request)
+        self.assertEqual(fingerprint, settings['FINGERPRINT'])
+
+
 if __name__ == "__main__":
     unittest.main()

From 407562b38b6ab375ae650c8799bdd511025527f4 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Thu, 9 Jun 2022 00:25:03 -0300
Subject: [PATCH 600/716] Drop Python 3.6 support (#5514)

* chore: Drop Python 3.6 support

* Attend PR comments

* Tweak versions

* Update dependencies version

* fix: Ubuntu workflow

* fix windows workflow

* chore: Remove comment

* update `install_requires` dependencies versions

* move lxml to main pinned requirements

* Attend code-review comments

* remove non-pinned 3.7 from windows workflow

* simplify condition

* lint

* remove paragraph

* refactor

* remove leftover
---
 .github/workflows/checks.yml        |  2 +-
 .github/workflows/tests-macos.yml   |  2 +-
 .github/workflows/tests-ubuntu.yml  | 11 ++++-------
 .github/workflows/tests-windows.yml |  5 +----
 README.rst                          |  2 +-
 docs/contributing.rst               | 10 +++++-----
 docs/intro/install.rst              |  4 ++--
 docs/topics/items.rst               |  5 -----
 docs/topics/media-pipeline.rst      |  2 +-
 scrapy/__init__.py                  |  4 ++--
 scrapy/utils/py36.py                | 11 -----------
 setup.py                            | 19 ++++++-------------
 tests/requirements.txt              |  4 +---
 tests/test_utils_python.py          |  8 +-------
 tox.ini                             | 23 ++++++++---------------
 15 files changed, 34 insertions(+), 78 deletions(-)
 delete mode 100644 scrapy/utils/py36.py

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 98fa44c7f..b26f344ff 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -19,7 +19,7 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: pylint
-        - python-version: 3.6
+        - python-version: 3.7
           env:
             TOXENV: typing
         - python-version: "3.10"  # Keep in sync with .readthedocs.yml
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 3aaf688c7..7819a4e12 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
 
     steps:
     - uses: actions/checkout@v2
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 1fc8d914b..be40c7c71 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -8,9 +8,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.7
-          env:
-            TOXENV: py
         - python-version: 3.8
           env:
             TOXENV: py
@@ -26,19 +23,19 @@ jobs:
         - python-version: pypy3
           env:
             TOXENV: pypy3
-            PYPY_VERSION: 3.6-v7.3.3
+            PYPY_VERSION: 3.9-v7.3.9
 
         # pinned deps
-        - python-version: 3.6.12
+        - python-version: 3.7.13
           env:
             TOXENV: pinned
-        - python-version: 3.6.12
+        - python-version: 3.7.13
           env:
             TOXENV: asyncio-pinned
         - python-version: pypy3
           env:
             TOXENV: pypy3-pinned
-            PYPY_VERSION: 3.6-v7.2.0
+            PYPY_VERSION: 3.7-v7.3.5
 
         # extras
         # extra-deps includes reppy, which does not support Python 3.9
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index ab7385118..955b9b449 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -8,12 +8,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.6
-          env:
-            TOXENV: windows-pinned
         - python-version: 3.7
           env:
-            TOXENV: py
+            TOXENV: windows-pinned
         - python-version: 3.8
           env:
             TOXENV: py
diff --git a/README.rst b/README.rst
index 6b563d638..b543a30f4 100644
--- a/README.rst
+++ b/README.rst
@@ -57,7 +57,7 @@ including a list of features.
 Requirements
 ============
 
-* Python 3.6+
+* Python 3.7+
 * Works on Linux, Windows, macOS, BSD
 
 Install
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 4d2580a6c..946bdc23e 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
 
 To run the tests on a specific :doc:`tox <tox:index>` environment, use
 ``-e <name>`` with an environment name from ``tox.ini``. For example, to run
-the tests with Python 3.6 use::
+the tests with Python 3.7 use::
 
-    tox -e py36
+    tox -e py37
 
 You can also specify a comma-separated list of environments, and use :ref:`tox’s
 parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
 parallel::
 
-    tox -e py36,py38 -p auto
+    tox -e py37,py38 -p auto
 
 To pass command-line options to :doc:`pytest <pytest:index>`, add them after
 ``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
@@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
     tox -- scrapy tests -x  # stop after first failure
 
 You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
-the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
+the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
 
-    tox -e py36 -- scrapy tests -n auto
+    tox -e py37 -- scrapy tests -n auto
 
 To see coverage report install :doc:`coverage <coverage:index>`
 (``pip install coverage``) and run:
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index b8d3a16bc..1f01c068d 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -9,8 +9,8 @@ Installation guide
 Supported Python versions
 =========================
 
-Scrapy requires Python 3.6+, either the CPython implementation (default) or
-the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
+Scrapy requires Python 3.7+, either the CPython implementation (default) or
+the PyPy 7.3.5+ implementation (see :ref:`python:implementations`).
 
 .. _intro-install-scrapy:
 
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 7cd482d07..167014381 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -102,11 +102,6 @@ Additionally, ``dataclass`` items also allow to:
 * define custom field metadata through :func:`dataclasses.field`, which can be used to
   :ref:`customize serialization <topics-exporters-field-serialization>`.
 
-They work natively in Python 3.7 or later, or using the `dataclasses
-backport`_ in Python 3.6.
-
-.. _dataclasses backport: https://pypi.org/project/dataclasses/
-
 Example::
 
     from dataclasses import dataclass
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 2513faae2..0925e6bb5 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
 can configure some extra functions like generating thumbnails and filtering
 the images based on their size.
 
-The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
+The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
 thumbnailing and normalizing images to JPEG/RGB format.
 
 .. _Pillow: https://github.com/python-pillow/Pillow
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 396f98219..86e584396 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 
 # Check minimum required Python version
-if sys.version_info < (3, 6):
-    print(f"Scrapy {__version__} requires Python 3.6+")
+if sys.version_info < (3, 7):
+    print(f"Scrapy {__version__} requires Python 3.7+")
     sys.exit(1)
 
 
diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py
deleted file mode 100644
index 653e2bbbb..000000000
--- a/scrapy/utils/py36.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import warnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncgen import collect_asyncgen  # noqa: F401
-
-
-warnings.warn(
-    "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.",
-    category=ScrapyDeprecationWarning,
-    stacklevel=2,
-)
diff --git a/setup.py b/setup.py
index d86c0f285..ed197273f 100644
--- a/setup.py
+++ b/setup.py
@@ -19,35 +19,29 @@ def has_environment_marker_platform_impl_support():
 
 
 install_requires = [
-    'Twisted>=17.9.0',
-    'cryptography>=2.0',
+    'Twisted>=18.9.0',
+    'cryptography>=2.8',
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
     'parsel>=1.5.0',
-    'pyOpenSSL>=16.2.0',
+    'pyOpenSSL>=19.1.0',
     'queuelib>=1.4.2',
     'service_identity>=16.0.0',
     'w3lib>=1.17.0',
-    'zope.interface>=4.1.3',
+    'zope.interface>=5.1.0',
     'protego>=0.1.15',
     'itemadapter>=0.1.0',
     'setuptools',
     'tldextract',
+    'lxml>=4.3.0',
 ]
 extras_require = {}
 cpython_dependencies = [
-    'lxml>=3.5.0',
     'PyDispatcher>=2.0.5',
 ]
 if has_environment_marker_platform_impl_support():
     extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies
     extras_require[':platform_python_implementation == "PyPy"'] = [
-        # Earlier lxml versions are affected by
-        # https://foss.heptapod.net/pypy/pypy/-/issues/2498,
-        # which was fixed in Cython 0.26, released on 2017-06-19, and used to
-        # generate the C headers of lxml release tarballs published since then, the
-        # first of which was:
-        'lxml>=4.0.0',
         'PyPyDispatcher>=2.1.0',
     ]
 else:
@@ -84,7 +78,6 @@ setup(
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
@@ -95,7 +88,7 @@ setup(
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    python_requires='>=3.6',
+    python_requires='>=3.7',
     install_requires=install_requires,
     extras_require=extras_require,
 )
diff --git a/tests/requirements.txt b/tests/requirements.txt
index d2a8aae1b..d9373dfa8 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,14 +1,12 @@
 # Tests requirements
 attrs
-dataclasses; python_version == '3.6'
 pyftpdlib
 pytest
 pytest-cov==3.0.0
 pytest-xdist
 sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures
-uvloop < 0.15.0; platform_system != "Windows" and python_version == '3.6'
-uvloop; platform_system != "Windows" and python_version > '3.6'
+uvloop; platform_system != "Windows"
 
 # optional for shell wrapper tests
 bpython
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 4b3964154..7dec5624a 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -3,7 +3,6 @@ import gc
 import operator
 import platform
 import unittest
-from datetime import datetime
 from itertools import count
 from warnings import catch_warnings, filterwarnings
 
@@ -224,12 +223,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         elif platform.python_implementation() == 'PyPy':
             self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
             self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
-
-            build_date = datetime.strptime(platform.python_build()[1], '%b %d %Y')
-            if build_date >= datetime(2020, 4, 7):  # PyPy 3.6-v7.3.1
-                self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
-            else:
-                self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
+            self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
 
     def test_without_none_values(self):
         self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
diff --git a/tox.ini b/tox.ini
index 6951b6d16..ab8a715c2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,15 +11,13 @@ minversion = 1.7.0
 deps =
     -rtests/requirements.txt
     # mitmproxy does not support PyPy
-    # mitmproxy does not support Windows when running Python < 3.7
     # Python 3.9+ requires mitmproxy >= 5.3.0
     # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
     #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
     # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
-    mitmproxy >= 4.0.4, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
-    mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
+    mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
     # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
-    markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy'
+    markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy'
     # Extras
     botocore>=1.4.87
 passenv =
@@ -44,7 +42,6 @@ deps =
     types-pyOpenSSL==20.0.3
     types-setuptools==57.0.0
 commands =
-    pip install types-dataclasses  # remove once py36 support is dropped
     mypy --show-error-codes {posargs: scrapy tests}
 
 [testenv:security]
@@ -75,18 +72,19 @@ commands =
 
 [pinned]
 deps =
-    cryptography==2.0
+    cryptography==2.8
     cssselect==0.9.1
     h2==3.0
     itemadapter==0.1.0
     parsel==1.5.0
     Protego==0.1.15
-    pyOpenSSL==16.2.0
+    pyOpenSSL==19.1.0
     queuelib==1.4.2
     service_identity==16.0.0
-    Twisted[http2]==17.9.0
+    Twisted[http2]==18.9.0
     w3lib==1.17.0
-    zope.interface==4.1.3
+    zope.interface==5.1.0
+    lxml==4.3.0
     -rtests/requirements.txt
 
     # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
@@ -95,7 +93,7 @@ deps =
     # Extras
     botocore==1.4.87
     google-cloud-storage==1.29.0
-    Pillow==4.0.0
+    Pillow==7.1.0
 setenv =
     _SCRAPY_PINNED=true
 install_command =
@@ -104,7 +102,6 @@ install_command =
 [testenv:pinned]
 deps =
     {[pinned]deps}
-    lxml==3.5.0
     PyDispatcher==2.0.5
 install_command = {[pinned]install_command}
 setenv =
@@ -114,9 +111,6 @@ setenv =
 basepython = python3
 deps =
     {[pinned]deps}
-    # First lxml version that includes a Windows wheel for Python 3.6, so we do
-    # not need to build lxml from sources in a CI Windows job:
-    lxml==3.8.0
     PyDispatcher==2.0.5
 install_command = {[pinned]install_command}
 setenv =
@@ -155,7 +149,6 @@ commands =
 basepython = {[testenv:pypy3]basepython}
 deps =
     {[pinned]deps}
-    lxml==4.0.0
     PyPyDispatcher==2.1.0
 commands = {[testenv:pypy3]commands}
 install_command = {[pinned]install_command}

From 2e6721fd86e3bd00301f8cd3ceb4175b2f395017 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Thu, 9 Jun 2022 08:37:01 -0300
Subject: [PATCH 601/716] docs: Update minimal versions that Scrapy is tested
 against

---
 docs/intro/install.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 1f01c068d..23c3af74b 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -54,9 +54,9 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
 
 The minimal versions which Scrapy is tested against are:
 
-* Twisted 14.0
-* lxml 3.4
-* pyOpenSSL 0.14
+* Twisted 18.9.0
+* lxml 4.3.0
+* pyOpenSSL 19.1.0
 
 Scrapy may work with older versions of these packages
 but it is not guaranteed it will continue working

From 6770d1ec62012fcfe8a36fdebeeb89cb5157c2df Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Thu, 9 Jun 2022 09:08:09 -0300
Subject: [PATCH 602/716] chore(tests): Remove validations for unsupported
 modules versions

---
 tests/test_downloadermiddleware.py | 20 +-------------------
 tests/test_utils_signal.py         | 20 --------------------
 tests/test_webclient.py            |  5 -----
 3 files changed, 1 insertion(+), 44 deletions(-)

diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index b538a0ed3..38be915f2 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,13 +1,11 @@
 import asyncio
-from unittest import mock, SkipTest
+from unittest import mock
 
 from pytest import mark
-from twisted import version as twisted_version
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
-from twisted.python.versions import Version
 
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
@@ -218,16 +216,6 @@ class MiddlewareUsingCoro(ManagerTestCase):
     """Middlewares using asyncio coroutines should work"""
 
     def test_asyncdef(self):
-        if (
-            self.reactor_pytest == 'asyncio'
-            and twisted_version < Version('twisted', 18, 4, 0)
-        ):
-            raise SkipTest(
-                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
-                'hangs when using AsyncIO and Twisted versions lower than '
-                '18.4.0'
-            )
-
         resp = Response('http://example.com/index.html')
 
         class CoroMiddleware:
@@ -248,12 +236,6 @@ class MiddlewareUsingCoro(ManagerTestCase):
 
     @mark.only_asyncio()
     def test_asyncdef_asyncio(self):
-        if twisted_version < Version('twisted', 18, 4, 0):
-            raise SkipTest(
-                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
-                'hangs when using Twisted versions lower than 18.4.0'
-            )
-
         resp = Response('http://example.com/index.html')
 
         class CoroMiddleware:
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index ad7394232..a36e7bc97 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -1,13 +1,10 @@
 import asyncio
-from unittest import SkipTest
 
 from pydispatch import dispatcher
 from pytest import mark
 from testfixtures import LogCapture
-from twisted import version as twisted_version
 from twisted.internet import defer, reactor
 from twisted.python.failure import Failure
-from twisted.python.versions import Version
 from twisted.trial import unittest
 
 from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
@@ -81,16 +78,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
         return "OK"
 
     def test_send_catch_log(self):
-        if (
-            self.reactor_pytest == 'asyncio'
-            and twisted_version < Version('twisted', 18, 4, 0)
-        ):
-            raise SkipTest(
-                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
-                'fails due to a timeout when using AsyncIO and Twisted '
-                'versions lower than 18.4.0'
-            )
-
         return super().test_send_catch_log()
 
 
@@ -104,13 +91,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
         return await get_from_asyncio_queue("OK")
 
     def test_send_catch_log(self):
-        if twisted_version < Version('twisted', 18, 4, 0):
-            raise SkipTest(
-                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
-                'fails due to a timeout when using Twisted versions lower '
-                'than 18.4.0'
-            )
-
         return super().test_send_catch_log()
 
 
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a6d55cb38..0d5827339 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -4,10 +4,7 @@ Tests borrowed from the twisted.web.client tests.
 """
 import os
 import shutil
-import sys
-from pkg_resources import parse_version
 
-import cryptography
 import OpenSSL.SSL
 from twisted.trial import unittest
 from twisted.web import server, static, util, resource
@@ -417,8 +414,6 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
         ).addCallback(self.assertEqual, to_bytes(s))
 
     def testPayloadDisabledCipher(self):
-        if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"):
-            self.skipTest("This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1")
         s = "0123456789" * 10
         settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'})
         client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)

From c4c5c9f25841a783aab2c682125f3200d6c6e446 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Thu, 9 Jun 2022 10:00:44 -0300
Subject: [PATCH 603/716] docs: Remove minimal versions paragraphs

---
 docs/intro/install.rst | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 23c3af74b..c1fd6d522 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -52,12 +52,6 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
 * `twisted`_, an asynchronous networking framework
 * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
 
-The minimal versions which Scrapy is tested against are:
-
-* Twisted 18.9.0
-* lxml 4.3.0
-* pyOpenSSL 19.1.0
-
 Scrapy may work with older versions of these packages
 but it is not guaranteed it will continue working
 because it’s not being tested against them.

From 197aca2c94201f9944404f30fc4a002309cad99b Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Thu, 9 Jun 2022 10:11:49 -0300
Subject: [PATCH 604/716] docs: Remove leftover

---
 docs/intro/install.rst | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index c1fd6d522..80a9c16d6 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -52,10 +52,6 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
 * `twisted`_, an asynchronous networking framework
 * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
 
-Scrapy may work with older versions of these packages
-but it is not guaranteed it will continue working
-because it’s not being tested against them.
-
 Some of these packages themselves depends on non-Python packages
 that might require additional installation steps depending on your platform.
 Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.

From ddfd192b704dddfefe2dd78345de239995a40159 Mon Sep 17 00:00:00 2001
From: Mohammadtaher Abbasi <mohammadtaher.abbasi@zoodfood.com>
Date: Sat, 11 Jun 2022 23:51:34 +0430
Subject: [PATCH 605/716] add tests for multiple headers with same name

---
 tests/test_http_headers.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 64ff7a73d..0c51fd701 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -38,6 +38,13 @@ class HeadersTest(unittest.TestCase):
         self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2'])
         assert h.getlist('X-Forwarded-For') is not hlist
 
+    def test_multivalue_for_one_header(self):
+        h = Headers((("a", "b"), ("a", "c")))
+        self.assertEqual(h["a"], b"c")
+        self.assertEqual(h.get("a"), b"c")
+        self.assertEqual(h.getlist("a"), [b"b", b"c"])
+        assert h.getlist("a") is not ["b", "c"]
+
     def test_encode_utf8(self):
         h = Headers({'key': '\xa3'}, encoding='utf-8')
         key, val = dict(h).popitem()

From 6a0bcf97cc6016cb966b92170709bc7518cf62c2 Mon Sep 17 00:00:00 2001
From: Mohammadtaher Abbasi <mohammadtaher.abbasi@zoodfood.com>
Date: Sat, 11 Jun 2022 23:52:21 +0430
Subject: [PATCH 606/716] Merge values of multiple headers with same name
 (#5515)

---
 scrapy/http/headers.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 1a2b99b0a..a9471d721 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,6 +1,7 @@
 from w3lib.http import headers_dict_to_raw
 from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.python import to_unicode
+from collections.abc import Mapping
 
 
 class Headers(CaselessDict):
@@ -10,6 +11,13 @@ class Headers(CaselessDict):
         self.encoding = encoding
         super().__init__(seq)
 
+    def update(self, seq):
+        seq = seq.items() if isinstance(seq, Mapping) else seq
+        iseq = {}
+        for k, v in seq:
+            iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
+        super().update(iseq)
+
     def normkey(self, key):
         """Normalize key to bytes"""
         return self._tobytes(key.title())
@@ -86,4 +94,5 @@ class Headers(CaselessDict):
 
     def __copy__(self):
         return self.__class__(self)
+
     copy = __copy__

From a135d6caf050f4b7b5af28d4cccc5d5ef51dbaf6 Mon Sep 17 00:00:00 2001
From: Mohammadtaher Abbasi <mtabba30@gmail.com>
Date: Mon, 13 Jun 2022 15:24:30 +0430
Subject: [PATCH 607/716] Move Mapping import line up
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/http/headers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index a9471d721..9c03fe54f 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,7 +1,8 @@
+from collections.abc import Mapping
+
 from w3lib.http import headers_dict_to_raw
 from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.python import to_unicode
-from collections.abc import Mapping
 
 
 class Headers(CaselessDict):

From 892c2a46554bdf80d49d3f28cc012c49cd1e19ca Mon Sep 17 00:00:00 2001
From: Mohammadtaher Abbasi <mohammadtaher.abbasi@zoodfood.com>
Date: Mon, 13 Jun 2022 23:46:42 +0430
Subject: [PATCH 608/716] delete unnecessary test

---
 tests/test_http_headers.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 0c51fd701..1ca936247 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -43,7 +43,6 @@ class HeadersTest(unittest.TestCase):
         self.assertEqual(h["a"], b"c")
         self.assertEqual(h.get("a"), b"c")
         self.assertEqual(h.getlist("a"), [b"b", b"c"])
-        assert h.getlist("a") is not ["b", "c"]
 
     def test_encode_utf8(self):
         h = Headers({'key': '\xa3'}, encoding='utf-8')

From 9e265a2c1f6bccb551e8292785e09462594b8402 Mon Sep 17 00:00:00 2001
From: Kromitvs <74136201+Kromitvs@users.noreply.github.com>
Date: Thu, 16 Jun 2022 19:52:19 +0100
Subject: [PATCH 609/716] Mind body to choose response class in cache, FTP and
 HTTP/1.0 (#4873)

---
 scrapy/core/downloader/handlers/ftp.py       |  6 +--
 scrapy/core/downloader/webclient.py          |  2 +-
 scrapy/extensions/httpcache.py               |  4 +-
 scrapy/responsetypes.py                      | 10 ++--
 tests/test_downloader_handlers.py            | 54 ++++++++++++++++++--
 tests/test_downloadermiddleware_httpcache.py | 15 ++++++
 tests/test_responsetypes.py                  |  2 +
 7 files changed, 78 insertions(+), 15 deletions(-)

diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 3ef129587..a495874bd 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -102,11 +102,11 @@ class FTPDownloadHandler:
 
     def _build_response(self, result, request, protocol):
         self.result = result
-        respcls = responsetypes.from_args(url=request.url)
         protocol.close()
-        body = protocol.filename or protocol.body.read()
         headers = {"local filename": protocol.filename or '', "size": protocol.size}
-        return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers)
+        body = to_bytes(protocol.filename or protocol.body.read())
+        respcls = responsetypes.from_args(url=request.url, body=body)
+        return respcls(url=request.url, status=200, body=body, headers=headers)
 
     def _failed(self, result, request):
         message = result.getErrorMessage()
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 06cb96489..7d048c1e4 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -112,7 +112,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
         request.meta['download_latency'] = self.headers_time - self.start_time
         status = int(self.status)
         headers = Headers(self.response_headers)
-        respcls = responsetypes.from_args(headers=headers, url=self._url)
+        respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
         return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
 
     def _set_connection_attributes(self, request):
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index c71484cfa..843e14812 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -240,7 +240,7 @@ class DbmCacheStorage:
         status = data['status']
         headers = Headers(data['headers'])
         body = data['body']
-        respcls = responsetypes.from_args(headers=headers, url=url)
+        respcls = responsetypes.from_args(headers=headers, url=url, body=body)
         response = respcls(url=url, headers=headers, status=status, body=body)
         return response
 
@@ -299,7 +299,7 @@ class FilesystemCacheStorage:
         url = metadata.get('response_url')
         status = metadata['status']
         headers = Headers(headers_raw_to_dict(rawheaders))
-        respcls = responsetypes.from_args(headers=headers, url=url)
+        respcls = responsetypes.from_args(headers=headers, url=url, body=body)
         response = respcls(url=url, headers=headers, status=status, body=body)
         return response
 
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 6ed9f8b8f..3efd4d2fd 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -95,12 +95,14 @@ class ResponseTypes:
         chunk = to_bytes(chunk)
         if not binary_is_text(chunk):
             return self.from_mimetype('application/octet-stream')
-        elif b"<html>" in chunk.lower():
+        lowercase_chunk = chunk.lower()
+        if b"<html>" in lowercase_chunk:
             return self.from_mimetype('text/html')
-        elif b"<?xml" in chunk.lower():
+        if b"<?xml" in lowercase_chunk:
             return self.from_mimetype('text/xml')
-        else:
-            return self.from_mimetype('text')
+        if b'<!doctype html>' in lowercase_chunk:
+            return self.from_mimetype('text/html')
+        return self.from_mimetype('text')
 
     def from_args(self, headers=None, url=None, filename=None, body=None):
         """Guess the most appropriate Response class based on
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 2bb53950d..72f52121e 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -25,7 +25,7 @@ from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.http import Headers, Request
+from scrapy.http import Headers, HtmlResponse, Request
 from scrapy.http.response.text import TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.spiders import Spider
@@ -389,6 +389,23 @@ class HttpTestCase(unittest.TestCase):
         d.addCallback(self.assertEqual, b'159')
         return d
 
+    def _test_response_class(self, filename, body, response_class):
+        def _test(response):
+            self.assertEqual(type(response), response_class)
+
+        request = Request(self.getURL(filename), body=body)
+        return self.download_request(request, Spider('foo')).addCallback(_test)
+
+    def test_response_class_from_url(self):
+        return self._test_response_class('foo.html', b'', HtmlResponse)
+
+    def test_response_class_from_body(self):
+        return self._test_response_class(
+            'foo',
+            b"<!DOCTYPE html>\n<title>.</title>",
+            HtmlResponse,
+        )
+
 
 class Http10TestCase(HttpTestCase):
     """HTTP 1.0 test case"""
@@ -971,6 +988,12 @@ class BaseFTPTestCase(unittest.TestCase):
     password = "passwd"
     req_meta = {"ftp_user": username, "ftp_password": password}
 
+    test_files = (
+        ('file.txt', b"I have the power!"),
+        ('file with spaces.txt', b"Moooooooooo power!"),
+        ('html-file-without-extension', b"<!DOCTYPE html>\n<title>.</title>"),
+    )
+
     def setUp(self):
         from twisted.protocols.ftp import FTPRealm, FTPFactory
         from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
@@ -981,8 +1004,8 @@ class BaseFTPTestCase(unittest.TestCase):
         userdir = os.path.join(self.directory, self.username)
         os.mkdir(userdir)
         fp = FilePath(userdir)
-        fp.child('file.txt').setContent(b"I have the power!")
-        fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
+        for filename, content in self.test_files:
+            fp.child(filename).setContent(content)
 
         # setup server
         realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory)
@@ -1069,6 +1092,27 @@ class BaseFTPTestCase(unittest.TestCase):
 
         return self._add_test_callbacks(d, _test)
 
+    def _test_response_class(self, filename, response_class):
+        f, local_fname = tempfile.mkstemp()
+        local_fname = to_bytes(local_fname)
+        os.close(f)
+        meta = {}
+        meta.update(self.req_meta)
+        request = Request(url=f"ftp://127.0.0.1:{self.portNum}/{filename}",
+                          meta=meta)
+        d = self.download_handler.download_request(request, None)
+
+        def _test(r):
+            self.assertEqual(type(r), response_class)
+            os.remove(local_fname)
+        return self._add_test_callbacks(d, _test)
+
+    def test_response_class_from_url(self):
+        return self._test_response_class('file.txt', TextResponse)
+
+    def test_response_class_from_body(self):
+        return self._test_response_class('html-file-without-extension', HtmlResponse)
+
 
 class FTPTestCase(BaseFTPTestCase):
 
@@ -1104,8 +1148,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
         os.mkdir(self.directory)
 
         fp = FilePath(self.directory)
-        fp.child('file.txt').setContent(b"I have the power!")
-        fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
+        for filename, content in self.test_files:
+            fp.child(filename).setContent(content)
 
         # setup server for anonymous access
         realm = FTPRealm(anonymousRoot=self.directory)
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 0c6dcf2aa..928c007f5 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -122,6 +122,21 @@ class DefaultStorageTest(_BaseTest):
             time.sleep(0.5)  # give the chance to expire
             assert storage.retrieve_response(self.spider, self.request)
 
+    def test_storage_no_content_type_header(self):
+        """Test that the response body is used to get the right response class
+        even if there is no Content-Type header"""
+        with self._storage() as storage:
+            assert storage.retrieve_response(self.spider, self.request) is None
+            response = Response(
+                'http://www.example.com',
+                body=b'<!DOCTYPE html>\n<title>.</title>',
+                status=202,
+            )
+            storage.store_response(self.spider, self.request, response)
+            cached_response = storage.retrieve_response(self.spider, self.request)
+            self.assertIsInstance(cached_response, HtmlResponse)
+            self.assertEqualResponse(response, cached_response)
+
 
 class DbmStorageTest(DefaultStorageTest):
 
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index c07d3a99c..4b4095fb0 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -54,6 +54,8 @@ class ResponseTypesTest(unittest.TestCase):
             (b'\x03\x02\xdf\xdd\x23', Response),
             (b'Some plain text\ndata with tabs\t and null bytes\0', TextResponse),
             (b'<html><head><title>Hello</title></head>', HtmlResponse),
+            # https://codersblock.com/blog/the-smallest-valid-html5-page/
+            (b'<!DOCTYPE html>\n<title>.</title>', HtmlResponse),
             (b'<?xml version="1.0" encoding="utf-8"', XmlResponse),
         ]
         for source, cls in mappings:

From 3e994bda45ac4f95cec0517c4b1be68a38314914 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 16 Jun 2022 20:53:14 +0200
Subject: [PATCH 610/716] Update for Python 3.7+

---
 docs/topics/exporters.rst   | 10 +---------
 scrapy/exporters.py         |  2 +-
 scrapy/settings/__init__.py |  9 ++++-----
 tests/test_exporters.py     |  5 ++---
 tests/test_feedexport.py    | 10 +++-------
 5 files changed, 11 insertions(+), 25 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 7580011ac..3c36ef002 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -205,7 +205,7 @@ BaseItemExporter
 
               ['field1', 'field2']
 
-      -   A dict [3]_ where keys are fields and values are output names::
+      -   A dict where keys are fields and values are output names::
 
               {'field1': 'Field 1', 'field2': 'Field 2'}
 
@@ -214,14 +214,6 @@ BaseItemExporter
              all their possible fields, exporters that do not support exporting
              a different subset of fields per item will only export the fields 
              found in the first item exported.
-      .. [3] Dicts preserve insertion order since `Python 3.7`_
-             (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version
-             of Python, use an OrderedDict_ to enforce a specific field order.
-
-             .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html
-             .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation
-             .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html
-             .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict
 
    .. attribute:: export_empty_fields
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index ad12f26d6..76cbe4d4b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -2,13 +2,13 @@
 Item Exporters are used to export/serialize items into different formats.
 """
 
-from collections import Mapping
 import csv
 import io
 import marshal
 import pickle
 import pprint
 import warnings
+from collections.abc import Mapping
 from xml.sax.saxutils import XMLGenerator
 
 from itemadapter import is_item, ItemAdapter
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 2bbe38481..6cacc63e1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,6 +1,5 @@
 import json
 import copy
-from collections import OrderedDict
 from collections.abc import MutableMapping
 from importlib import import_module
 from pprint import pformat
@@ -199,7 +198,7 @@ class BaseSettings(MutableMapping):
         return dict(value)
 
     def getdictorlist(self, name, default=None):
-        """Get a setting value as either an ``OrderedDict`` or a list.
+        """Get a setting value as either a :class:`dict` or a :class:`list`.
 
         If the setting is already a dict or a list, a copy of it will be
         returned.
@@ -209,7 +208,7 @@ class BaseSettings(MutableMapping):
 
         For example, settings populated from the command line will return:
 
-        -   ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to
+        -   ``{'key1': 'value1', 'key2': 'value2'}`` if set to
             ``'{"key1": "value1", "key2": "value2"}'``
 
         -   ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
@@ -222,10 +221,10 @@ class BaseSettings(MutableMapping):
         """
         value = self.get(name, default)
         if value is None:
-            return OrderedDict()
+            return {}
         if isinstance(value, str):
             try:
-                return json.loads(value, object_pairs_hook=OrderedDict)
+                return json.loads(value)
             except ValueError:
                 return value.split(',')
         return copy.deepcopy(value)
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 6ba7428f6..096cd3116 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -4,7 +4,6 @@ import marshal
 import pickle
 import tempfile
 import unittest
-from collections import OrderedDict
 from io import BytesIO
 from datetime import datetime
 from warnings import catch_warnings, filterwarnings
@@ -114,11 +113,11 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(name, 'John\xa3')
 
         ie = self._get_exporter(
-            fields_to_export=OrderedDict([('name', u'名稱')])
+            fields_to_export={'name': '名稱'}
         )
         self.assertEqual(
             list(ie._get_serialized_fields(self.i)),
-            [(u'名稱', u'John\xa3')]
+            [('名稱', 'John\xa3')]
         )
 
     def test_field_custom_serializer(self):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 83aabbdc7..9098e035d 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -11,7 +11,7 @@ import sys
 import tempfile
 import warnings
 from abc import ABC, abstractmethod
-from collections import defaultdict, OrderedDict
+from collections import defaultdict
 from contextlib import ExitStack
 from io import BytesIO
 from logging import getLogger
@@ -998,9 +998,7 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_export_items_field_names(self):
         items = [{'foo': 'bar'}]
-        header = OrderedDict((
-            ("foo", "Foo"),
-        ))
+        header = {'foo': 'Foo'}
         rows = [{'Foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': header}
         yield self.assertExported(items, list(header.values()), rows,
@@ -1023,9 +1021,7 @@ class FeedExportTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_export_items_json_field_names(self):
         items = [{'foo': 'bar'}]
-        header = OrderedDict((
-            ("foo", "Foo"),
-        ))
+        header = {'foo': 'Foo'}
         rows = [{'Foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
         yield self.assertExported(items, list(header.values()), rows,

From 1b9ed22becf03311ec014dc9b7e0c09ce87b612c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 08:27:17 +0200
Subject: [PATCH 611/716] Remove Python < 3.7 leftover

---
 tests/test_feedexport.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 9098e035d..946c94bd4 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1004,8 +1004,6 @@ class FeedExportTest(FeedExportTestBase):
         yield self.assertExported(items, list(header.values()), rows,
                                   settings=settings)
 
-    @pytest.mark.skipif(sys.version_info < (3, 7),
-                        reason='Only official in Python 3.7+')
     @defer.inlineCallbacks
     def test_export_items_dict_field_names(self):
         items = [{'foo': 'bar'}]

From 24f382fa459434cccfa4c0a8884a48d09d75e243 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 08:31:45 +0200
Subject: [PATCH 612/716] test_feedexport: remove ordered=False

---
 tests/test_feedexport.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 946c94bd4..4006b5957 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -657,8 +657,8 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         return data
 
     @defer.inlineCallbacks
-    def assertExported(self, items, header, rows, settings=None, ordered=True):
-        yield self.assertExportedCsv(items, header, rows, settings, ordered)
+    def assertExported(self, items, header, rows, settings=None):
+        yield self.assertExportedCsv(items, header, rows, settings)
         yield self.assertExportedJsonLines(items, rows, settings)
         yield self.assertExportedXml(items, rows, settings)
         yield self.assertExportedPickle(items, rows, settings)
@@ -719,7 +719,7 @@ class FeedExportTest(FeedExportTestBase):
         return content
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+    def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {
@@ -730,10 +730,7 @@ class FeedExportTest(FeedExportTestBase):
 
         reader = csv.DictReader(to_unicode(data['csv']).splitlines())
         got_rows = list(reader)
-        if ordered:
-            self.assertEqual(reader.fieldnames, header)
-        else:
-            self.assertEqual(set(reader.fieldnames), set(header))
+        self.assertEqual(reader.fieldnames, header)
 
         self.assertEqual(rows, got_rows)
 
@@ -886,7 +883,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
         ]
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows, ordered=False)
+        yield self.assertExported(items, header, rows)
 
     @defer.inlineCallbacks
     def test_export_no_items_not_store_empty(self):
@@ -958,7 +955,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam4', 'foo': '', 'baz': ''},
         ]
         rows_jl = [dict(row) for row in items]
-        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
+        yield self.assertExportedCsv(items, header, rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -968,7 +965,7 @@ class FeedExportTest(FeedExportTestBase):
         header = ["foo"]
         rows = [{'foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': []}
-        yield self.assertExportedCsv(items, header, rows, ordered=False)
+        yield self.assertExportedCsv(items, header, rows)
         yield self.assertExportedJsonLines(items, rows, settings)
 
     @defer.inlineCallbacks
@@ -1146,7 +1143,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam', 'foo': 'bar'}
         ]
         rows_jl = items
-        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False)
+        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -2065,7 +2062,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+    def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {

From 3729c6d26698ae6b8a7ef297606a1c7630d82619 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 08:33:34 +0200
Subject: [PATCH 613/716] Remove unused import and redundant import

---
 tests/test_feedexport.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 4006b5957..8ef221b70 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -21,7 +21,6 @@ from unittest import mock
 from urllib.parse import urljoin, quote
 from urllib.request import pathname2url
 
-import pytest
 import lxml.etree
 from testfixtures import LogCapture
 from twisted.internet import defer
@@ -731,7 +730,6 @@ class FeedExportTest(FeedExportTestBase):
         reader = csv.DictReader(to_unicode(data['csv']).splitlines())
         got_rows = list(reader)
         self.assertEqual(reader.fieldnames, header)
-
         self.assertEqual(rows, got_rows)
 
     @defer.inlineCallbacks
@@ -1815,7 +1813,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_lzma_plugin_filters(self):
-        import sys
         if "PyPy" in sys.version:
             # https://foss.heptapod.net/pypy/pypy/-/issues/3527
             raise unittest.SkipTest("lzma filters doesn't work in PyPy")

From 6e878490e823a8105276b71ca5f6dc789465d330 Mon Sep 17 00:00:00 2001
From: Michel Ace <divtiply@gmail.com>
Date: Fri, 17 Jun 2022 08:37:14 +0200
Subject: [PATCH 614/716] Support and prefer the .jsonl file extension (#4848)

---
 docs/intro/overview.rst             | 4 ++--
 docs/intro/tutorial.rst             | 2 +-
 docs/topics/feed-exports.rst        | 3 ++-
 docs/topics/item-pipeline.rst       | 8 ++++----
 scrapy/settings/default_settings.py | 1 +
 5 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index f3d652621..cfa6bfa83 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -45,9 +45,9 @@ https://quotes.toscrape.com, following the pagination::
 Put this in a text file, name it to something like ``quotes_spider.py``
 and run the spider using the :command:`runspider` command::
 
-    scrapy runspider quotes_spider.py -o quotes.jl
+    scrapy runspider quotes_spider.py -o quotes.jsonl
 
-When this finishes you will have in the ``quotes.jl`` file a list of the
+When this finishes you will have in the ``quotes.jsonl`` file a list of the
 quotes in JSON Lines format, containing text and author, looking like this::
 
     {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index cde1b1ef4..75928077e 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -482,7 +482,7 @@ to append new content to any existing file. However, appending to a JSON file
 makes the file contents invalid JSON. When appending to a file, consider
 using a different serialization format, such as `JSON Lines`_::
 
-    scrapy crawl quotes -o quotes.jl
+    scrapy crawl quotes -o quotes.jsonl
 
 The `JSON Lines`_ format is useful because it's stream-like, you can easily
 append new records to it. It doesn't have the same problem of JSON when you run
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9a13eb82f..398f80633 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -638,6 +638,7 @@ Default::
     {
         'json': 'scrapy.exporters.JsonItemExporter',
         'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+        'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
         'jl': 'scrapy.exporters.JsonLinesItemExporter',
         'csv': 'scrapy.exporters.CsvItemExporter',
         'xml': 'scrapy.exporters.XmlItemExporter',
@@ -763,7 +764,7 @@ source spider in the feed URI:
 
 #.  Use ``%(spider_name)s`` in your feed URI::
 
-        scrapy crawl <spider_name> -o "%(spider_name)s.jl"
+        scrapy crawl <spider_name> -o "%(spider_name)s.jsonl"
 
 
 .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 882ff5661..af294f52c 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -99,11 +99,11 @@ contain a price::
                 raise DropItem(f"Missing price in {item}")
 
 
-Write items to a JSON file
---------------------------
+Write items to a JSON lines file
+--------------------------------
 
 The following pipeline stores all scraped items (from all spiders) into a
-single ``items.jl`` file, containing one item per line serialized in JSON
+single ``items.jsonl`` file, containing one item per line serialized in JSON
 format::
 
    import json
@@ -113,7 +113,7 @@ format::
    class JsonWriterPipeline:
 
        def open_spider(self, spider):
-           self.file = open('items.jl', 'w')
+           self.file = open('items.jsonl', 'w')
 
        def close_spider(self, spider):
            self.file.close()
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index f5a3efe69..ff86af125 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -154,6 +154,7 @@ FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',
     'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+    'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
     'jl': 'scrapy.exporters.JsonLinesItemExporter',
     'csv': 'scrapy.exporters.CsvItemExporter',
     'xml': 'scrapy.exporters.XmlItemExporter',

From 516e2d6ec0da77b8e0c01eb5188311b5fbeaa22e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 08:55:45 +0200
Subject: [PATCH 615/716] Revert "test_feedexport: remove ordered=False"

This reverts commit 24f382fa459434cccfa4c0a8884a48d09d75e243.
---
 tests/test_feedexport.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8ef221b70..fe90501fb 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -656,8 +656,8 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         return data
 
     @defer.inlineCallbacks
-    def assertExported(self, items, header, rows, settings=None):
-        yield self.assertExportedCsv(items, header, rows, settings)
+    def assertExported(self, items, header, rows, settings=None, ordered=True):
+        yield self.assertExportedCsv(items, header, rows, settings, ordered)
         yield self.assertExportedJsonLines(items, rows, settings)
         yield self.assertExportedXml(items, rows, settings)
         yield self.assertExportedPickle(items, rows, settings)
@@ -718,7 +718,7 @@ class FeedExportTest(FeedExportTestBase):
         return content
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None):
+    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
         settings.update({
             'FEEDS': {
@@ -729,7 +729,11 @@ class FeedExportTest(FeedExportTestBase):
 
         reader = csv.DictReader(to_unicode(data['csv']).splitlines())
         got_rows = list(reader)
-        self.assertEqual(reader.fieldnames, header)
+        if ordered:
+            self.assertEqual(reader.fieldnames, header)
+        else:
+            self.assertEqual(set(reader.fieldnames), set(header))
+
         self.assertEqual(rows, got_rows)
 
     @defer.inlineCallbacks
@@ -881,7 +885,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
         ]
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows)
+        yield self.assertExported(items, header, rows, ordered=False)
 
     @defer.inlineCallbacks
     def test_export_no_items_not_store_empty(self):
@@ -953,7 +957,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam4', 'foo': '', 'baz': ''},
         ]
         rows_jl = [dict(row) for row in items]
-        yield self.assertExportedCsv(items, header, rows_csv)
+        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -963,7 +967,7 @@ class FeedExportTest(FeedExportTestBase):
         header = ["foo"]
         rows = [{'foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': []}
-        yield self.assertExportedCsv(items, header, rows)
+        yield self.assertExportedCsv(items, header, rows, ordered=False)
         yield self.assertExportedJsonLines(items, rows, settings)
 
     @defer.inlineCallbacks
@@ -1141,7 +1145,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam', 'foo': 'bar'}
         ]
         rows_jl = items
-        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv)
+        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -2059,7 +2063,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None):
+    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
         settings.update({
             'FEEDS': {

From bc285f393ca8ff33ef715f98ef3367c973d23ab3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 09:00:39 +0200
Subject: [PATCH 616/716] Revert "Revert "test_feedexport: remove
 ordered=False""

This reverts commit 516e2d6ec0da77b8e0c01eb5188311b5fbeaa22e.
---
 tests/test_feedexport.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index fe90501fb..8ef221b70 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -656,8 +656,8 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         return data
 
     @defer.inlineCallbacks
-    def assertExported(self, items, header, rows, settings=None, ordered=True):
-        yield self.assertExportedCsv(items, header, rows, settings, ordered)
+    def assertExported(self, items, header, rows, settings=None):
+        yield self.assertExportedCsv(items, header, rows, settings)
         yield self.assertExportedJsonLines(items, rows, settings)
         yield self.assertExportedXml(items, rows, settings)
         yield self.assertExportedPickle(items, rows, settings)
@@ -718,7 +718,7 @@ class FeedExportTest(FeedExportTestBase):
         return content
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+    def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {
@@ -729,11 +729,7 @@ class FeedExportTest(FeedExportTestBase):
 
         reader = csv.DictReader(to_unicode(data['csv']).splitlines())
         got_rows = list(reader)
-        if ordered:
-            self.assertEqual(reader.fieldnames, header)
-        else:
-            self.assertEqual(set(reader.fieldnames), set(header))
-
+        self.assertEqual(reader.fieldnames, header)
         self.assertEqual(rows, got_rows)
 
     @defer.inlineCallbacks
@@ -885,7 +881,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
         ]
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows, ordered=False)
+        yield self.assertExported(items, header, rows)
 
     @defer.inlineCallbacks
     def test_export_no_items_not_store_empty(self):
@@ -957,7 +953,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam4', 'foo': '', 'baz': ''},
         ]
         rows_jl = [dict(row) for row in items]
-        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
+        yield self.assertExportedCsv(items, header, rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -967,7 +963,7 @@ class FeedExportTest(FeedExportTestBase):
         header = ["foo"]
         rows = [{'foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': []}
-        yield self.assertExportedCsv(items, header, rows, ordered=False)
+        yield self.assertExportedCsv(items, header, rows)
         yield self.assertExportedJsonLines(items, rows, settings)
 
     @defer.inlineCallbacks
@@ -1145,7 +1141,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam', 'foo': 'bar'}
         ]
         rows_jl = items
-        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False)
+        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -2063,7 +2059,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+    def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {

From ec5cf3e9cea3c66aca4cf1aad576f33edca3ad1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 09:10:18 +0200
Subject: [PATCH 617/716] test_feedexport: solve ordered comparison issues

---
 tests/test_feedexport.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8ef221b70..ec48f8d4a 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -726,11 +726,9 @@ class FeedExportTest(FeedExportTestBase):
             },
         })
         data = yield self.exported_data(items, settings)
-
         reader = csv.DictReader(to_unicode(data['csv']).splitlines())
-        got_rows = list(reader)
-        self.assertEqual(reader.fieldnames, header)
-        self.assertEqual(rows, got_rows)
+        self.assertEqual(reader.fieldnames, list(header))
+        self.assertEqual(rows, list(reader))
 
     @defer.inlineCallbacks
     def assertExportedJsonLines(self, items, rows, settings=None):
@@ -1141,7 +1139,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam', 'foo': 'bar'}
         ]
         rows_jl = items
-        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv)
+        yield self.assertExportedCsv(items, ['foo', 'egg'], rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks

From 4ef71829b22b7362d08d6897090595138107852f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 10:37:27 +0200
Subject: [PATCH 618/716] If TWISTED_REACTOR is None, reuse any pre-installed
 reactor (#5528)

---
 docs/topics/settings.rst                      |  7 ++-
 scrapy/crawler.py                             |  3 +-
 scrapy/utils/reactor.py                       |  2 +-
 tests/CrawlerProcess/reactor_default.py       | 17 +++++
 .../reactor_default_twisted_reactor_select.py | 20 ++++++
 tests/CrawlerProcess/reactor_select.py        | 19 ++++++
 ..._select_subclass_twisted_reactor_select.py | 31 +++++++++
 .../reactor_select_twisted_reactor_select.py  | 22 +++++++
 tests/test_crawler.py                         | 63 +++++++++++++++++--
 9 files changed, 172 insertions(+), 12 deletions(-)
 create mode 100644 tests/CrawlerProcess/reactor_default.py
 create mode 100644 tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
 create mode 100644 tests/CrawlerProcess/reactor_select.py
 create mode 100644 tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
 create mode 100644 tests/CrawlerProcess/reactor_select_twisted_reactor_select.py

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4e105642d..f3b28c4c4 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1638,9 +1638,10 @@ which raises :exc:`Exception`, becomes::
 
 
 The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
-means that Scrapy will install the default reactor defined by Twisted for the
-current platform. This is to maintain backward compatibility and avoid possible
-problems caused by using a non-default reactor.
+means that Scrapy will use the existing reactor if one is already installed, or
+install the default reactor defined by Twisted for the current platform. This
+is to maintain backward compatibility and avoid possible problems caused by
+using a non-default reactor.
 
 For additional information, see :doc:`core/howto/choosing-reactor`.
 
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index d669d93a8..dcf0c2146 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -78,8 +78,7 @@ class Crawler:
             if reactor_class:
                 install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
             else:
-                from twisted.internet import default
-                default.install()
+                from twisted.internet import reactor  # noqa: F401
             log_reactor_info()
         if reactor_class:
             verify_installed_reactor(reactor_class)
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 96395543c..bc543b230 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -83,7 +83,7 @@ def verify_installed_reactor(reactor_path):
     path."""
     from twisted.internet import reactor
     reactor_class = load_object(reactor_path)
-    if not isinstance(reactor, reactor_class):
+    if not reactor.__class__ == reactor_class:
         msg = ("The installed reactor "
                f"({reactor.__module__}.{reactor.__class__.__name__}) does not "
                f"match the requested one ({reactor_path})")
diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py
new file mode 100644
index 000000000..5a21a3717
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_default.py
@@ -0,0 +1,17 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet import reactor
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()
+
diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
new file mode 100644
index 000000000..c476722ef
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
@@ -0,0 +1,20 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet import reactor
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
+
+
diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py
new file mode 100644
index 000000000..eac6e2f89
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_select.py
@@ -0,0 +1,19 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet import selectreactor
+selectreactor.install()
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()
+
+
diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
new file mode 100644
index 000000000..47f480605
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
@@ -0,0 +1,31 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet.main import installReactor
+from twisted.internet.selectreactor import SelectReactor
+
+
+class SelectReactorSubclass(SelectReactor):
+    pass
+
+
+reactor = SelectReactorSubclass()
+installReactor(reactor)
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
+
+
+
+
diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
new file mode 100644
index 000000000..e0d2dab26
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
@@ -0,0 +1,22 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet import selectreactor
+selectreactor.install()
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
+
+
+
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 957525382..1ff2e8a67 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -308,6 +308,57 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
         self.assertNotIn("ReactorAlreadyInstalledError", log)
 
+    def test_reactor_default(self):
+        log = self.run_script('reactor_default.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertNotIn("ReactorAlreadyInstalledError", log)
+
+    def test_reactor_default_twisted_reactor_select(self):
+        log = self.run_script('reactor_default_twisted_reactor_select.py')
+        if platform.system() == 'Windows':
+            # The goal of this test function is to test that, when a reactor is
+            # installed (the default one here) and a different reactor is
+            # configured (select here), an error raises.
+            #
+            # In Windows the default reactor is the select reactor, so that
+            # error does not raise.
+            #
+            # If that ever becomes the case on more platforms (i.e. if Linux
+            # also starts using the select reactor by default in a future
+            # version of Twisted), then we will need to rethink this test.
+            self.assertIn('Spider closed (finished)', log)
+        else:
+            self.assertNotIn('Spider closed (finished)', log)
+            self.assertIn(
+                (
+                    "does not match the requested one "
+                    "(twisted.internet.selectreactor.SelectReactor)"
+                ),
+                log,
+            )
+
+    def test_reactor_select(self):
+        log = self.run_script('reactor_select.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertNotIn("ReactorAlreadyInstalledError", log)
+
+    def test_reactor_select_twisted_reactor_select(self):
+        log = self.run_script('reactor_select_twisted_reactor_select.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertNotIn("ReactorAlreadyInstalledError", log)
+
+    def test_reactor_select_subclass_twisted_reactor_select(self):
+        log = self.run_script('reactor_select_subclass_twisted_reactor_select.py')
+        self.assertNotIn('Spider closed (finished)', log)
+        self.assertIn(
+            (
+                "does not match the requested one "
+                "(twisted.internet.selectreactor.SelectReactor)"
+            ),
+            log,
+        )
+
     def test_asyncio_enabled_no_reactor(self):
         log = self.run_script('asyncio_enabled_no_reactor.py')
         self.assertIn('Spider closed (finished)', log)
@@ -340,33 +391,33 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             self.assertNotIn("TimeoutError", log)
             self.assertNotIn("twisted.internet.error.DNSLookupError", log)
 
-    def test_reactor_select(self):
+    def test_twisted_reactor_select(self):
         log = self.run_script("twisted_reactor_select.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
 
     @mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows")
-    def test_reactor_poll(self):
+    def test_twisted_reactor_poll(self):
         log = self.run_script("twisted_reactor_poll.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
 
-    def test_reactor_asyncio(self):
+    def test_twisted_reactor_asyncio(self):
         log = self.run_script("twisted_reactor_asyncio.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    def test_reactor_asyncio_custom_settings(self):
+    def test_twisted_reactor_asyncio_custom_settings(self):
         log = self.run_script("twisted_reactor_custom_settings.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    def test_reactor_asyncio_custom_settings_same(self):
+    def test_twisted_reactor_asyncio_custom_settings_same(self):
         log = self.run_script("twisted_reactor_custom_settings_same.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    def test_reactor_asyncio_custom_settings_conflict(self):
+    def test_twisted_reactor_asyncio_custom_settings_conflict(self):
         log = self.run_script("twisted_reactor_custom_settings_conflict.py")
         self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
         self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log)

From 54bfb9649bdec565f9798cc41643ed1bae25bd67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 17 Jun 2022 11:51:02 +0200
Subject: [PATCH 619/716] Cover #5525 in the 2.6.2 release notes (#5535)

---
 docs/conf.py  | 1 +
 docs/news.rst | 9 +++++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 55aa72d5a..9a0afe73e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -295,6 +295,7 @@ intersphinx_mapping = {
     'twisted': ('https://twistedmatrix.com/documents/current', None),
     'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
 }
+intersphinx_disabled_reftypes = []
 
 
 # Options for sphinx-hoverxref options
diff --git a/docs/news.rst b/docs/news.rst
index ffeb50390..7993b4b4f 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5,14 +5,19 @@ Release notes
 
 .. _release-2.6.2:
 
-Scrapy 2.6.2 (2022-0?-??)
--------------------------
+Scrapy 2.6.2 (to be determined)
+-------------------------------
 
 Fixes additional regressions introduced in 2.6.0:
 
 -   :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple
     spiders (:issue:`5435`, :issue:`5436`)
 
+-   Installing a Twisted reactor before Scrapy does (e.g. importing
+    :mod:`twisted.internet.reactor` somewhere at the module level) no longer
+    prevents Scrapy from starting, as long as a different reactor is not
+    specified in :setting:`TWISTED_REACTOR` (:issue:`5525`, :issue:`5528`)
+
 -   Fixed an exception that was being logged after the spider finished under
     certain conditions (:issue:`5437`, :issue:`5440`)
 

From e3e69d1209407c72a6478936bdbfd32cc22e9432 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 20 Jun 2022 11:46:13 +0200
Subject: [PATCH 620/716] Pin documentation requirements (#5536)

---
 docs/requirements.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index a0930ba1e..9f9aef711 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,4 +1,4 @@
-Sphinx>=3.0
-sphinx-hoverxref>=0.2b1
-sphinx-notfound-page>=0.4
-sphinx-rtd-theme>=0.5.2
\ No newline at end of file
+sphinx==5.0.2
+sphinx-hoverxref==1.1.1
+sphinx-notfound-page==0.8
+sphinx-rtd-theme==1.0.0

From d8223adfacc7e0ae684e5f9463474707bfcb008d Mon Sep 17 00:00:00 2001
From: Emanuele <eman_93@hotmail.it>
Date: Mon, 20 Jun 2022 11:54:05 +0200
Subject: [PATCH 621/716] =?UTF-8?q?Typo:=20cleanup=20(verb)=20=E2=86=92=20?=
 =?UTF-8?q?clean=20up=20(#5538)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/README.rst b/docs/README.rst
index 0b7afa548..36dd5aea4 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -43,7 +43,7 @@ This command will fire up your default browser and open the main page of your
 Start over
 ----------
 
-To cleanup all generated documentation files and start from scratch run::
+To clean up all generated documentation files and start from scratch run::
 
     make clean
 

From 387326fad42c4709933851108f2370d3105b8dd1 Mon Sep 17 00:00:00 2001
From: Vardhaman <83634399+cyai@users.noreply.github.com>
Date: Thu, 23 Jun 2022 14:40:49 +0530
Subject: [PATCH 622/716] MAINT: Updated f-string format

Updated the code with the f-string method for better and cleaner understanding.
---
 scrapy/downloadermiddlewares/cookies.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 3afa06077..c592acb57 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -104,8 +104,8 @@ class CookiesMiddleware:
         for key in ("name", "value", "path", "domain"):
             if cookie.get(key) is None:
                 if key in ("name", "value"):
-                    msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
-                    logger.warning(msg.format(request, cookie, key))
+                    msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
+                    logger.warning(msg)
                     return
                 continue
             if isinstance(cookie[key], (bool, float, int, str)):

From c4c816624fc4fd5fbc1866c507b661e92704136a Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Thu, 30 Jun 2022 10:42:01 -0300
Subject: [PATCH 623/716] chore: Deprecate the
 `scrapy.downloadermiddlewares.decompression` module

---
 scrapy/downloadermiddlewares/decompression.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 0fcf8fb8c..98f18a836 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -9,10 +9,19 @@ import tarfile
 import zipfile
 from io import BytesIO
 from tempfile import mktemp
+import warnings
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.responsetypes import responsetypes
 
 
+warnings.warn(
+    'scrapy.downloadermiddlewares.decompression is deprecated',
+    ScrapyDeprecationWarning,
+    stacklevel=2,
+)
+
+
 logger = logging.getLogger(__name__)
 
 

From fe08a119d965b2291e44801901e15b58a1f959ad Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Thu, 30 Jun 2022 10:46:00 -0300
Subject: [PATCH 624/716] chore: import only used function

---
 scrapy/downloadermiddlewares/decompression.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 98f18a836..e01e9cc76 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -9,13 +9,13 @@ import tarfile
 import zipfile
 from io import BytesIO
 from tempfile import mktemp
-import warnings
+from warnings import warn
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.responsetypes import responsetypes
 
 
-warnings.warn(
+warn(
     'scrapy.downloadermiddlewares.decompression is deprecated',
     ScrapyDeprecationWarning,
     stacklevel=2,

From 09c3a4ad082dd6fc431be65975292d2eba369ad6 Mon Sep 17 00:00:00 2001
From: Rotzbua <Rotzbua@users.noreply.github.com>
Date: Tue, 12 Jul 2022 12:41:46 +0200
Subject: [PATCH 625/716] Fix doc: `scrapy.exporter` to `scrapy.exporters`

---
 docs/topics/exporters.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 3c36ef002..9360ecf37 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -117,7 +117,7 @@ after your custom code.
 
 Example::
 
-      from scrapy.exporter import XmlItemExporter
+      from scrapy.exporters import XmlItemExporter
 
       class ProductXmlExporter(XmlItemExporter):
 

From 1c7ed4f2e59a94651d6d9d136cab78821cd3e80e Mon Sep 17 00:00:00 2001
From: Rotzbua <Rotzbua@users.noreply.github.com>
Date: Thu, 6 Jan 2022 22:21:56 +0100
Subject: [PATCH 626/716] [doc] Remove incompatible web service project

* Abandoned since 2017
* Not compatible with Python3
---
 docs/index.rst             |  4 ----
 docs/topics/webservice.rst | 11 -----------
 2 files changed, 15 deletions(-)
 delete mode 100644 docs/topics/webservice.rst

diff --git a/docs/index.rst b/docs/index.rst
index 75e08f537..40c6cb485 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -130,7 +130,6 @@ Built-in services
    topics/stats
    topics/email
    topics/telnetconsole
-   topics/webservice
 
 :doc:`topics/logging`
     Learn how to use Python's builtin logging on Scrapy.
@@ -144,9 +143,6 @@ Built-in services
 :doc:`topics/telnetconsole`
     Inspect a running crawler using a built-in Python console.
 
-:doc:`topics/webservice`
-    Monitor and control a crawler using a web service.
-
 
 Solving specific problems
 =========================
diff --git a/docs/topics/webservice.rst b/docs/topics/webservice.rst
deleted file mode 100644
index 2c4052c04..000000000
--- a/docs/topics/webservice.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-.. _topics-webservice:
-
-===========
-Web Service
-===========
-
-webservice has been moved into a separate project.
-
-It is hosted at:
-
-    https://github.com/scrapy-plugins/scrapy-jsonrpc

From 2f13f23d927900de0a89197ded0ee7aed387e351 Mon Sep 17 00:00:00 2001
From: Ikko Ashimine <eltociear@gmail.com>
Date: Fri, 15 Jul 2022 18:16:23 +0900
Subject: [PATCH 627/716] Fix typo in sep-014.rst

requets -> requests
---
 sep/sep-014.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sep/sep-014.rst b/sep/sep-014.rst
index 0859e3f7c..2521aa0e5 100644
--- a/sep/sep-014.rst
+++ b/sep/sep-014.rst
@@ -590,11 +590,11 @@ Request Generator
 
        def generate_requests(self, response):
            """
-           Extract and process new requets from response
+           Extract and process new requests from response
            """
            requests = []
            for ext in self._request_extractors:
-               requets.extend(ext.extract_requests(response))
+               requests.extend(ext.extract_requests(response))
 
            for proc in self._request_processors:
                requests = proc(requests)

From 9b33b82a8b802c3906c2f1eaf1b88efee9b2fb09 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sun, 17 Jul 2022 15:50:40 +0500
Subject: [PATCH 628/716] Fixed intersphinx references

---
 docs/conf.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 378b01804..3241295af 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -291,9 +291,9 @@ intersphinx_mapping = {
     'pytest': ('https://docs.pytest.org/en/latest', None),
     'python': ('https://docs.python.org/3', None),
     'sphinx': ('https://www.sphinx-doc.org/en/master', None),
-    'tox': ('https://tox.readthedocs.io/en/latest', None),
-    'twisted': ('https://twistedmatrix.com/documents/current', None),
-    'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
+    'tox': ('https://tox.wiki/en/latest/', None),
+    'twisted': ('https://docs.twisted.org/en/stable/', None),
+    'twistedapi': ('https://docs.twisted.org/en/stable/api/', None),
     'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
 }
 intersphinx_disabled_reftypes = []

From 26c70318cb14806a07ee09d0283e9d5d306490e9 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sun, 17 Jul 2022 16:47:20 +0500
Subject: [PATCH 629/716] make Scrapy testing suite more robust in environments
 where non-existing hosts are resolvable

---
 tests/__init__.py                 | 10 ++++++++++
 tests/test_command_shell.py       |  4 +++-
 tests/test_crawl.py               |  4 ++++
 tests/test_downloader_handlers.py |  5 ++++-
 4 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index 12ce79fa9..bb62851dc 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -5,6 +5,7 @@ see https://docs.scrapy.org/en/latest/contributing.html#running-tests
 """
 
 import os
+import socket
 
 # ignore system-wide proxies for tests
 # which would send requests to a totally unsuspecting server
@@ -25,6 +26,15 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
                              'sample_data')
 
 
+# In some environments accessing a non-existing host doesn't raise an
+# error. In such cases we're going to skip tests which rely on it.
+try:
+    socket.getaddrinfo('non-existing-host', 80)
+    NON_EXISTING_RESOLVABLE = True
+except socket.gaierror:
+    NON_EXISTING_RESOLVABLE = False
+
+
 def get_testdata(*paths):
     """Return test data"""
     path = os.path.join(tests_datadir, *paths)
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 16c9559b5..33189e9be 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -6,7 +6,7 @@ from twisted.internet import defer
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
 
-from tests import tests_datadir
+from tests import tests_datadir, NON_EXISTING_RESOLVABLE
 
 
 class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@@ -109,6 +109,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_dns_failures(self):
+        if NON_EXISTING_RESOLVABLE:
+            raise unittest.SkipTest("Non-existing hosts are resolvable")
         url = 'www.somedomainthatdoesntexi.st'
         errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False)
         self.assertEqual(errcode, 1, out or err)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 7bda3bef2..f9ffcd6bb 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -3,6 +3,7 @@ import logging
 from ipaddress import IPv4Address
 from socket import gethostbyname
 from urllib.parse import urlparse
+import unittest
 
 from pytest import mark
 from testfixtures import LogCapture
@@ -17,6 +18,7 @@ from scrapy.exceptions import StopDownload
 from scrapy.http import Request
 from scrapy.http.response import Response
 from scrapy.utils.python import to_unicode
+from tests import NON_EXISTING_RESOLVABLE
 from tests.mockserver import MockServer
 from tests.spiders import (
     AsyncDefAsyncioGenComplexSpider,
@@ -137,6 +139,8 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_retry_dns_error(self):
+        if NON_EXISTING_RESOLVABLE:
+            raise unittest.SkipTest("Non-existing hosts are resolvable")
         crawler = self.runner.create_crawler(SimpleSpider)
         with LogCapture() as log:
             # try to fetch the homepage of a non-existent domain
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 72f52121e..883960084 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -4,7 +4,7 @@ import shutil
 import sys
 import tempfile
 from typing import Optional, Type
-from unittest import mock
+from unittest import mock, SkipTest
 
 from testfixtures import LogCapture
 from twisted.cred import checkers, credentials, portal
@@ -32,6 +32,7 @@ from scrapy.spiders import Spider
 from scrapy.utils.misc import create_instance
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler, skip_if_no_boto
+from tests import NON_EXISTING_RESOLVABLE
 from tests.mockserver import (
     Echo,
     ForeverTakingResource,
@@ -791,6 +792,8 @@ class Http11ProxyTestCase(HttpProxyTestCase):
     @defer.inlineCallbacks
     def test_download_with_proxy_https_timeout(self):
         """ Test TunnelingTCP4ClientEndpoint """
+        if NON_EXISTING_RESOLVABLE:
+            raise SkipTest("Non-existing hosts are resolvable")
         http_proxy = self.getURL('')
         domain = 'https://no-such-domain.nosuch'
         request = Request(

From e248360e6e3dbb36fab185caf131707195fa6a26 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 18 Jul 2022 23:49:08 +0500
Subject: [PATCH 630/716] remove compatibility code from tests for the case
 dataclasses module is not available

It was Python 3.6 compat code, and Python 3.6 support is dropped.
---
 tests/test_exporters.py       | 32 ++++++++++++++---------------
 tests/test_loader.py          | 23 +++++++--------------
 tests/test_pipeline_files.py  | 37 ++++++++++++----------------------
 tests/test_pipeline_images.py | 38 ++++++++++++-----------------------
 tests/test_utils_serialize.py | 18 +++++++----------
 5 files changed, 55 insertions(+), 93 deletions(-)

diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 096cd3116..69ac928c3 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -4,6 +4,7 @@ import marshal
 import pickle
 import tempfile
 import unittest
+import dataclasses
 from io import BytesIO
 from datetime import datetime
 from warnings import catch_warnings, filterwarnings
@@ -21,31 +22,30 @@ from scrapy.exporters import (
 )
 
 
+def custom_serializer(value):
+    return str(int(value) + 2)
+
+
 class TestItem(Item):
     name = Field()
     age = Field()
 
 
-def custom_serializer(value):
-    return str(int(value) + 2)
-
-
 class CustomFieldItem(Item):
     name = Field()
     age = Field(serializer=custom_serializer)
 
 
-try:
-    from dataclasses import make_dataclass, field
-except ImportError:
-    TestDataClass = None
-    CustomFieldDataclass = None
-else:
-    TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)])
-    CustomFieldDataclass = make_dataclass(
-        "CustomFieldDataclass",
-        [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))]
-    )
+@dataclasses.dataclass
+class TestDataClass:
+    name: str
+    age: int
+
+
+@dataclasses.dataclass
+class CustomFieldDataclass:
+    name: str
+    age: int = dataclasses.field(metadata={"serializer": custom_serializer})
 
 
 class BaseItemExporterTest(unittest.TestCase):
@@ -54,8 +54,6 @@ class BaseItemExporterTest(unittest.TestCase):
     custom_field_item_class = CustomFieldItem
 
     def setUp(self):
-        if self.item_class is None:
-            raise unittest.SkipTest("item class is None")
         self.i = self.item_class(name='John\xa3', age='22')
         self.output = BytesIO()
         self.ie = self._get_exporter()
diff --git a/tests/test_loader.py b/tests/test_loader.py
index f7ab1f236..c0937b349 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,4 +1,5 @@
 import unittest
+import dataclasses
 
 import attr
 from itemadapter import ItemAdapter
@@ -10,13 +11,6 @@ from scrapy.loader import ItemLoader
 from scrapy.selector import Selector
 
 
-try:
-    from dataclasses import make_dataclass, field as dataclass_field
-except ImportError:
-    make_dataclass = None
-    dataclass_field = None
-
-
 # test items
 class NameItem(Item):
     name = Field()
@@ -41,6 +35,11 @@ class AttrsNameItem:
     name = attr.ib(default="")
 
 
+@dataclasses.dataclass
+class TestDataClass:
+    name: list = dataclasses.field(default_factory=list)
+
+
 # test item loaders
 class NameItemLoader(ItemLoader):
     default_item_class = TestItem
@@ -187,16 +186,8 @@ class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase
     item_class = AttrsNameItem
 
 
-@unittest.skipIf(not make_dataclass, "dataclasses module is not available")
 class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase):
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        if make_dataclass:
-            self.item_class = make_dataclass(
-                "TestDataClass",
-                [("name", list, dataclass_field(default_factory=list))],
-            )
+    item_class = TestDataClass
 
 
 class BaseNoInputReprocessingLoader(ItemLoader):
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 4228173ed..5d381c018 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -7,6 +7,7 @@ from shutil import rmtree
 from tempfile import mkdtemp
 from unittest import mock, skipIf
 from urllib.parse import urlparse
+import dataclasses
 
 import attr
 from itemadapter import ItemAdapter
@@ -32,13 +33,6 @@ from scrapy.utils.test import (
 )
 
 
-try:
-    from dataclasses import make_dataclass, field as dataclass_field
-except ImportError:
-    make_dataclass = None
-    dataclass_field = None
-
-
 def _mocked_download_func(request, info):
     response = request.meta.get('response')
     return response() if callable(response) else response
@@ -226,24 +220,19 @@ class FilesPipelineTestCaseFieldsItem(FilesPipelineTestCaseFieldsMixin, unittest
     item_class = FilesPipelineTestItem
 
 
-@skipIf(not make_dataclass, "dataclasses module is not available")
-class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+@dataclasses.dataclass
+class FilesPipelineTestDataClass:
+    name: str
+    # default fields
+    file_urls: list = dataclasses.field(default_factory=list)
+    files: list = dataclasses.field(default_factory=list)
+    # overridden fields
+    custom_file_urls: list = dataclasses.field(default_factory=list)
+    custom_files: list = dataclasses.field(default_factory=list)
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        if make_dataclass:
-            self.item_class = make_dataclass(
-                "FilesPipelineTestDataClass",
-                [
-                    ("name", str),
-                    # default fields
-                    ("file_urls", list, dataclass_field(default_factory=list)),
-                    ("files", list, dataclass_field(default_factory=list)),
-                    # overridden fields
-                    ("custom_file_urls", list, dataclass_field(default_factory=list)),
-                    ("custom_files", list, dataclass_field(default_factory=list)),
-                ],
-            )
+
+class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = FilesPipelineTestDataClass
 
 
 @attr.s
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index dd94d296b..e6f5bea21 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -4,6 +4,7 @@ import random
 from shutil import rmtree
 from tempfile import mkdtemp
 from unittest import skipIf
+import dataclasses
 
 import attr
 from itemadapter import ItemAdapter
@@ -16,13 +17,6 @@ from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
 
 
-try:
-    from dataclasses import make_dataclass, field as dataclass_field
-except ImportError:
-    make_dataclass = None
-    dataclass_field = None
-
-
 try:
     from PIL import Image
 except ImportError:
@@ -203,25 +197,19 @@ class ImagesPipelineTestCaseFieldsItem(ImagesPipelineTestCaseFieldsMixin, unitte
     item_class = ImagesPipelineTestItem
 
 
-@skipIf(not make_dataclass, "dataclasses module is not available")
-class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
-    item_class = None
+@dataclasses.dataclass
+class ImagesPipelineTestDataClass:
+    name: str
+    # default fields
+    image_urls: list = dataclasses.field(default_factory=list)
+    images: list = dataclasses.field(default_factory=list)
+    # overridden fields
+    custom_image_urls: list = dataclasses.field(default_factory=list)
+    custom_images: list = dataclasses.field(default_factory=list)
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        if make_dataclass:
-            self.item_class = make_dataclass(
-                "FilesPipelineTestDataClass",
-                [
-                    ("name", str),
-                    # default fields
-                    ("image_urls", list, dataclass_field(default_factory=list)),
-                    ("images", list, dataclass_field(default_factory=list)),
-                    # overridden fields
-                    ("custom_image_urls", list, dataclass_field(default_factory=list)),
-                    ("custom_images", list, dataclass_field(default_factory=list)),
-                ],
-            )
+
+class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = ImagesPipelineTestDataClass
 
 
 @attr.s
diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py
index daf022aee..a51de1877 100644
--- a/tests/test_utils_serialize.py
+++ b/tests/test_utils_serialize.py
@@ -1,6 +1,7 @@
 import datetime
 import json
 import unittest
+import dataclasses
 from decimal import Decimal
 
 import attr
@@ -10,12 +11,6 @@ from scrapy.http import Request, Response
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
 
-try:
-    from dataclasses import make_dataclass
-except ImportError:
-    make_dataclass = None
-
-
 class JsonEncoderTestCase(unittest.TestCase):
 
     def setUp(self):
@@ -56,12 +51,13 @@ class JsonEncoderTestCase(unittest.TestCase):
         self.assertIn(r.url, rs)
         self.assertIn(str(r.status), rs)
 
-    @unittest.skipIf(not make_dataclass, "No dataclass support")
     def test_encode_dataclass_item(self):
-        TestDataClass = make_dataclass(
-            "TestDataClass",
-            [("name", str), ("url", str), ("price", int)],
-        )
+        @dataclasses.dataclass
+        class TestDataClass:
+            name: str
+            url: str
+            price: int
+
         item = TestDataClass(name="Product", url="http://product.org", price=1)
         encoded = self.encoder.encode(item)
         self.assertEqual(

From 105468959363ee50b597038ac30fd32d3ea1b1f2 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Mon, 18 Jul 2022 23:53:30 +0500
Subject: [PATCH 631/716] remove unused imports

thanks flake8!
---
 tests/test_pipeline_files.py  | 2 +-
 tests/test_pipeline_images.py | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 5d381c018..d641e7a43 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -5,7 +5,7 @@ from datetime import datetime
 from io import BytesIO
 from shutil import rmtree
 from tempfile import mkdtemp
-from unittest import mock, skipIf
+from unittest import mock
 from urllib.parse import urlparse
 import dataclasses
 
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index e6f5bea21..0082e7a4e 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -3,7 +3,6 @@ import io
 import random
 from shutil import rmtree
 from tempfile import mkdtemp
-from unittest import skipIf
 import dataclasses
 
 import attr

From b103664bf45b079e5488b13a0737866de1b7dc50 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Tue, 19 Jul 2022 20:39:26 +0500
Subject: [PATCH 632/716] Address 2/3 of warnings from tests (#5561)

---
 pytest.ini                                   |   3 +
 tests/test_contracts.py                      |   4 +-
 tests/test_crawl.py                          | 100 +++----
 tests/test_crawler.py                        |  36 ++-
 tests/test_downloadermiddleware_httpauth.py  |  14 +-
 tests/test_downloadermiddleware_httpproxy.py |  12 +-
 tests/test_downloadermiddleware_stats.py     |   7 +-
 tests/test_dupefilters.py                    |  27 +-
 tests/test_engine.py                         | 228 ++++++++--------
 tests/test_engine_stop_download_bytes.py     |  32 ++-
 tests/test_engine_stop_download_headers.py   |  32 ++-
 tests/test_feedexport.py                     | 269 +++++++------------
 tests/test_logformatter.py                   |   6 +-
 tests/test_pipeline_crawl.py                 |  12 +-
 tests/test_request_attribute_binding.py      |  21 +-
 tests/test_request_cb_kwargs.py              |   5 +-
 tests/test_scheduler.py                      |   3 +-
 tests/test_scheduler_base.py                 |  15 +-
 tests/test_spiderloader/__init__.py          |   5 +-
 tests/test_utils_project.py                  |  27 +-
 tests/test_utils_request.py                  |  13 +-
 tests/test_utils_response.py                 |  21 +-
 22 files changed, 424 insertions(+), 468 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index ae2ed2029..af0f2fb6e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,3 +21,6 @@ addopts =
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
     only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
+filterwarnings =
+    ignore:scrapy.downloadermiddlewares.decompression is deprecated
+    ignore:Module scrapy.utils.reqser is deprecated
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index d0f4a68c2..136056f50 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -5,11 +5,11 @@ from twisted.python import failure
 from twisted.trial import unittest
 
 from scrapy import FormRequest
-from scrapy.crawler import CrawlerRunner
 from scrapy.spidermiddlewares.httperror import HttpError
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.item import Item, Field
+from scrapy.utils.test import get_crawler
 from scrapy.contracts import ContractsManager, Contract
 from scrapy.contracts.default import (
     UrlContract,
@@ -398,7 +398,7 @@ class ContractsManagerTest(unittest.TestCase):
             TestSameUrlSpider.parse_first.__doc__ = contract_doc
             TestSameUrlSpider.parse_second.__doc__ = contract_doc
 
-            crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
+            crawler = get_crawler(TestSameUrlSpider)
             yield crawler.crawl()
 
         self.assertEqual(crawler.spider.visited, 2)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index f9ffcd6bb..59c271868 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -18,6 +18,7 @@ from scrapy.exceptions import StopDownload
 from scrapy.http import Request
 from scrapy.http.response import Response
 from scrapy.utils.python import to_unicode
+from scrapy.utils.test import get_crawler
 from tests import NON_EXISTING_RESOLVABLE
 from tests.mockserver import MockServer
 from tests.spiders import (
@@ -49,14 +50,13 @@ class CrawlTestCase(TestCase):
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
-        self.runner = CrawlerRunner()
 
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
 
     @defer.inlineCallbacks
     def test_follow_all(self):
-        crawler = self.runner.create_crawler(FollowAllSpider)
+        crawler = get_crawler(FollowAllSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(len(crawler.spider.urls_visited), 11)  # 10 + start_url
 
@@ -79,7 +79,7 @@ class CrawlTestCase(TestCase):
 
         settings = {"DOWNLOAD_DELAY": delay,
                     'RANDOMIZE_DOWNLOAD_DELAY': randomize}
-        crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
+        crawler = get_crawler(FollowAllSpider, settings)
         yield crawler.crawl(**crawl_kwargs)
         times = crawler.spider.times
         total_time = times[-1] - times[0]
@@ -92,7 +92,7 @@ class CrawlTestCase(TestCase):
         # of ``total`` and ``delay`` values that are too small for the test
         # code above to have any meaning.
         settings["DOWNLOAD_DELAY"] = 0
-        crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
+        crawler = get_crawler(FollowAllSpider, settings)
         yield crawler.crawl(**crawl_kwargs)
         times = crawler.spider.times
         total_time = times[-1] - times[0]
@@ -102,7 +102,7 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_success(self):
-        crawler = self.runner.create_crawler(DelaySpider)
+        crawler = get_crawler(DelaySpider)
         yield crawler.crawl(n=0.5, mockserver=self.mockserver)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 > 0)
@@ -110,7 +110,7 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_failure(self):
-        crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider)
+        crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
         yield crawler.crawl(n=0.5, mockserver=self.mockserver)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 == 0)
@@ -125,14 +125,14 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_retry_503(self):
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
         self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_retry_conn_failed(self):
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)
         self._assert_retried(log)
@@ -141,7 +141,7 @@ class CrawlTestCase(TestCase):
     def test_retry_dns_error(self):
         if NON_EXISTING_RESOLVABLE:
             raise unittest.SkipTest("Non-existing hosts are resolvable")
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             # try to fetch the homepage of a non-existent domain
             yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
@@ -150,7 +150,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):
         with LogCapture('scrapy', level=logging.ERROR) as log:
-            crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
+            crawler = get_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
 
         self.assertEqual(len(log.records), 1)
@@ -161,7 +161,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_bug_yielding(self):
         with LogCapture('scrapy', level=logging.ERROR) as log:
-            crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
+            crawler = get_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
 
         self.assertEqual(len(log.records), 1)
@@ -172,7 +172,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_lazyness(self):
         settings = {"CONCURRENT_REQUESTS": 1}
-        crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
+        crawler = get_crawler(BrokenStartRequestsSpider, settings)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertTrue(
             crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
@@ -181,7 +181,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_dupes(self):
         settings = {"CONCURRENT_REQUESTS": 1}
-        crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider)
+        crawler = get_crawler(DuplicateStartRequestsSpider, settings)
         yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver)
         self.assertEqual(crawler.spider.visited, 6)
 
@@ -210,7 +210,7 @@ Connection: close
 foo body
 with multiples lines
 '''})
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver)
         self.assertEqual(str(log).count("Got response 200"), 1)
@@ -218,7 +218,7 @@ with multiples lines
     @defer.inlineCallbacks
     def test_retry_conn_lost(self):
         # connection lost after receiving data
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)
         self._assert_retried(log)
@@ -226,7 +226,7 @@ with multiples lines
     @defer.inlineCallbacks
     def test_retry_conn_aborted(self):
         # connection lost before receiving data
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)
         self._assert_retried(log)
@@ -245,7 +245,7 @@ with multiples lines
         req0.meta['next'] = req1
         req1.meta['next'] = req2
         req2.meta['next'] = req3
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=req0, mockserver=self.mockserver)
         # basic asserts in case of weird communication errors
         self.assertIn('responses', crawler.spider.meta)
@@ -271,7 +271,7 @@ with multiples lines
         def cb(response):
             est.append(get_engine_status(crawler.engine))
 
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
         self.assertEqual(len(est), 1, est)
         s = dict(est[0])
@@ -286,7 +286,7 @@ with multiples lines
         def cb(response):
             est.append(format_engine_status(crawler.engine))
 
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
         self.assertEqual(len(est), 1, est)
         est = est[0].split("\n")[2:-2]  # remove header & footer
@@ -317,7 +317,7 @@ with multiples lines
             def start_requests(self):
                 raise TestError
 
-        crawler = self.runner.create_crawler(FaultySpider)
+        crawler = get_crawler(FaultySpider)
         yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError)
         self.assertFalse(crawler.crawling)
 
@@ -328,26 +328,28 @@ with multiples lines
                 "tests.pipelines.ZeroDivisionErrorPipeline": 300,
             }
         }
-        crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider, settings)
         yield self.assertFailure(
-            self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
+            crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
             ZeroDivisionError)
         self.assertFalse(crawler.crawling)
 
     @defer.inlineCallbacks
     def test_crawlerrunner_accepts_crawler(self):
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
+        runner = CrawlerRunner()
         with LogCapture() as log:
-            yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+            yield runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
         self.assertIn("Got response 200", str(log))
 
     @defer.inlineCallbacks
     def test_crawl_multiple(self):
-        self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
-        self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
+        runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})
+        runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
 
         with LogCapture() as log:
-            yield self.runner.join()
+            yield runner.join()
 
         self._assert_retried(log)
         self.assertIn("Got response 200", str(log))
@@ -358,7 +360,6 @@ class CrawlSpiderTestCase(TestCase):
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
-        self.runner = CrawlerRunner()
 
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
@@ -370,7 +371,7 @@ class CrawlSpiderTestCase(TestCase):
         def _on_item_scraped(item):
             items.append(item)
 
-        crawler = self.runner.create_crawler(spider_cls)
+        crawler = get_crawler(spider_cls)
         crawler.signals.connect(_on_item_scraped, signals.item_scraped)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@@ -378,10 +379,9 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_crawlspider_with_parse(self):
-        self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver)
-
+        crawler = get_crawler(CrawlSpiderWithParseMethod)
         with LogCapture() as log:
-            yield self.runner.join()
+            yield crawler.crawl(mockserver=self.mockserver)
 
         self.assertIn("[parse] status 200 (foo: None)", str(log))
         self.assertIn("[parse] status 201 (foo: None)", str(log))
@@ -389,10 +389,9 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_crawlspider_with_errback(self):
-        self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)
-
+        crawler = get_crawler(CrawlSpiderWithErrback)
         with LogCapture() as log:
-            yield self.runner.join()
+            yield crawler.crawl(mockserver=self.mockserver)
 
         self.assertIn("[parse] status 200 (foo: None)", str(log))
         self.assertIn("[parse] status 201 (foo: None)", str(log))
@@ -403,18 +402,19 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_async_def_parse(self):
-        self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        crawler = get_crawler(AsyncDefSpider)
         with LogCapture() as log:
-            yield self.runner.join()
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
         self.assertIn("Got response 200", str(log))
 
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_asyncio_parse(self):
-        runner = CrawlerRunner({"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"})
-        runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        crawler = get_crawler(AsyncDefAsyncioSpider, {
+            "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+        })
         with LogCapture() as log:
-            yield runner.join()
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
         self.assertIn("Got response 200", str(log))
 
     @mark.only_asyncio()
@@ -433,7 +433,7 @@ class CrawlSpiderTestCase(TestCase):
         def _on_item_scraped(item):
             items.append(item)
 
-        crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider)
+        crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider)
         crawler.signals.connect(_on_item_scraped, signals.item_scraped)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@@ -479,14 +479,14 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_response_ssl_certificate_none(self):
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url("/echo?body=test", is_secure=False)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta['responses'][0].certificate)
 
     @defer.inlineCallbacks
     def test_response_ssl_certificate(self):
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url("/echo?body=test", is_secure=True)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         cert = crawler.spider.meta['responses'][0].certificate
@@ -497,7 +497,7 @@ class CrawlSpiderTestCase(TestCase):
     @mark.xfail(reason="Responses with no body return early and contain no certificate")
     @defer.inlineCallbacks
     def test_response_ssl_certificate_empty_response(self):
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url("/status?n=200", is_secure=True)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         cert = crawler.spider.meta['responses'][0].certificate
@@ -507,7 +507,7 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_dns_server_ip_address_none(self):
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url('/status?n=200')
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         ip_address = crawler.spider.meta['responses'][0].ip_address
@@ -515,7 +515,7 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_dns_server_ip_address(self):
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url('/echo?body=test')
         expected_netloc, _ = urlparse(url).netloc.split(':')
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
@@ -525,7 +525,7 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_bytes_received_stop_download_callback(self):
-        crawler = self.runner.create_crawler(BytesReceivedCallbackSpider)
+        crawler = get_crawler(BytesReceivedCallbackSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("failure"))
         self.assertIsInstance(crawler.spider.meta["response"], Response)
@@ -534,7 +534,7 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_bytes_received_stop_download_errback(self):
-        crawler = self.runner.create_crawler(BytesReceivedErrbackSpider)
+        crawler = get_crawler(BytesReceivedErrbackSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("response"))
         self.assertIsInstance(crawler.spider.meta["failure"], Failure)
@@ -549,7 +549,7 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_headers_received_stop_download_callback(self):
-        crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider)
+        crawler = get_crawler(HeadersReceivedCallbackSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("failure"))
         self.assertIsInstance(crawler.spider.meta["response"], Response)
@@ -557,7 +557,7 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_headers_received_stop_download_errback(self):
-        crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider)
+        crawler = get_crawler(HeadersReceivedErrbackSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("response"))
         self.assertIsInstance(crawler.spider.meta["failure"], Failure)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index f7aa769e4..d67abed7c 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -13,11 +13,13 @@ from twisted.trial import unittest
 
 import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
+from scrapy.utils.test import get_crawler
 from scrapy.extensions.throttle import AutoThrottle
 from scrapy.extensions import telnet
 from scrapy.utils.test import get_testenv
@@ -34,9 +36,6 @@ class BaseCrawlerTest(unittest.TestCase):
 
 class CrawlerTestCase(BaseCrawlerTest):
 
-    def setUp(self):
-        self.crawler = Crawler(DefaultSpider, Settings())
-
     def test_populate_spidercls_settings(self):
         spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
         project_settings = {'TEST1': 'project', 'TEST3': 'project'}
@@ -46,7 +45,9 @@ class CrawlerTestCase(BaseCrawlerTest):
 
         settings = Settings()
         settings.setdict(project_settings, priority='project')
-        crawler = Crawler(CustomSettingsSpider, settings)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+            crawler = Crawler(CustomSettingsSpider, settings)
 
         self.assertEqual(crawler.settings.get('TEST1'), 'spider')
         self.assertEqual(crawler.settings.get('TEST2'), 'spider')
@@ -56,12 +57,14 @@ class CrawlerTestCase(BaseCrawlerTest):
         self.assertTrue(crawler.settings.frozen)
 
     def test_crawler_accepts_dict(self):
-        crawler = Crawler(DefaultSpider, {'foo': 'bar'})
+        crawler = get_crawler(DefaultSpider, {'foo': 'bar'})
         self.assertEqual(crawler.settings['foo'], 'bar')
         self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
 
     def test_crawler_accepts_None(self):
-        crawler = Crawler(DefaultSpider)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+            crawler = Crawler(DefaultSpider)
         self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
 
     def test_crawler_rejects_spider_objects(self):
@@ -77,7 +80,7 @@ class SpiderSettingsTestCase(unittest.TestCase):
                 'AUTOTHROTTLE_ENABLED': True
             }
 
-        crawler = Crawler(MySpider, {})
+        crawler = get_crawler(MySpider)
         enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
         self.assertIn(AutoThrottle, enabled_exts)
 
@@ -91,7 +94,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
         class MySpider(scrapy.Spider):
             name = 'spider'
 
-        Crawler(MySpider, {})
+        get_crawler(MySpider)
         assert get_scrapy_root_handler() is None
 
     def test_spider_custom_settings_log_level(self):
@@ -111,7 +114,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
 
         configure_logging()
         self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
-        crawler = Crawler(MySpider, {})
+        crawler = get_crawler(MySpider)
         self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
         info_count = crawler.stats.get_value('log_count/INFO')
         logging.debug('debug message')
@@ -148,7 +151,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
             }
 
         configure_logging()
-        Crawler(MySpider, {})
+        get_crawler(MySpider)
         logging.debug('debug message')
 
         with open(log_file, 'rb') as fo:
@@ -229,22 +232,25 @@ class NoRequestsSpider(scrapy.Spider):
 @mark.usefixtures('reactor_pytest')
 class CrawlerRunnerHasSpider(unittest.TestCase):
 
+    def _runner(self):
+        return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})
+
     @defer.inlineCallbacks
     def test_crawler_runner_bootstrap_successful(self):
-        runner = CrawlerRunner()
+        runner = self._runner()
         yield runner.crawl(NoRequestsSpider)
         self.assertEqual(runner.bootstrap_failed, False)
 
     @defer.inlineCallbacks
     def test_crawler_runner_bootstrap_successful_for_several(self):
-        runner = CrawlerRunner()
+        runner = self._runner()
         yield runner.crawl(NoRequestsSpider)
         yield runner.crawl(NoRequestsSpider)
         self.assertEqual(runner.bootstrap_failed, False)
 
     @defer.inlineCallbacks
     def test_crawler_runner_bootstrap_failed(self):
-        runner = CrawlerRunner()
+        runner = self._runner()
 
         try:
             yield runner.crawl(ExceptionSpider)
@@ -257,7 +263,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_crawler_runner_bootstrap_failed_for_several(self):
-        runner = CrawlerRunner()
+        runner = self._runner()
 
         try:
             yield runner.crawl(ExceptionSpider)
@@ -275,12 +281,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         if self.reactor_pytest == 'asyncio':
             CrawlerRunner(settings={
                 "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",
             })
         else:
             msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
             with self.assertRaisesRegex(Exception, msg):
                 runner = CrawlerRunner(settings={
                     "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                    "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",
                 })
                 yield runner.crawl(NoRequestsSpider)
 
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index 0362e2018..b9f3e24a4 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -1,7 +1,9 @@
 import unittest
 
+import pytest
 from w3lib.http import basic_auth_header
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
 from scrapy.spiders import Spider
@@ -30,8 +32,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
         self.spider = TestSpiderLegacy('foo')
 
     def test_auth(self):
-        mw = HttpAuthMiddleware()
-        mw.spider_opened(self.spider)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):
+            mw = HttpAuthMiddleware()
+            mw.spider_opened(self.spider)
 
         # initial request, sets the domain and sends the header
         req = Request('http://example.com/')
@@ -49,8 +53,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
         self.assertNotIn('Authorization', req.headers)
 
     def test_auth_already_set(self):
-        mw = HttpAuthMiddleware()
-        mw.spider_opened(self.spider)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):
+            mw = HttpAuthMiddleware()
+            mw.spider_opened(self.spider)
         req = Request('http://example.com/',
                       headers=dict(Authorization='Digest 123'))
         assert mw.process_request(req, self.spider) is None
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 7c97bf32a..4ac85c1ec 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -1,13 +1,13 @@
 import os
-from functools import partial
+
+import pytest
 from twisted.trial.unittest import TestCase
 
 from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Request
 from scrapy.spiders import Spider
-from scrapy.crawler import Crawler
-from scrapy.settings import Settings
+from scrapy.utils.test import get_crawler
 
 spider = Spider('foo')
 
@@ -23,9 +23,9 @@ class TestHttpProxyMiddleware(TestCase):
         os.environ = self._oldenv
 
     def test_not_enabled(self):
-        settings = Settings({'HTTPPROXY_ENABLED': False})
-        crawler = Crawler(Spider, settings)
-        self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))
+        crawler = get_crawler(Spider, {'HTTPPROXY_ENABLED': False})
+        with pytest.raises(NotConfigured):
+            HttpProxyMiddleware.from_crawler(crawler)
 
     def test_no_environment_proxies(self):
         os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 9e75f0a50..7d88ba4d2 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -1,7 +1,9 @@
+import warnings
 from itertools import product
 from unittest import TestCase
 
 from scrapy.downloadermiddlewares.stats import DownloaderStats
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.response import response_httprepr
@@ -54,7 +56,10 @@ class TestDownloaderStats(TestCase):
         for test_response in test_responses:
             self.crawler.stats.set_value('downloader/response_bytes', 0)
             self.mw.process_response(self.req, test_response, self.spider)
-            self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response)))
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+                resp_size = len(response_httprepr(test_response))
+            self.assertStatsEqual('downloader/response_bytes', resp_size)
 
     def test_process_exception(self):
         self.mw.process_exception(self.req, MyException(), self.spider)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index b7df2554a..8a37a8ebe 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -10,7 +10,6 @@ from scrapy.dupefilters import RFPDupeFilter
 from scrapy.http import Request
 from scrapy.core.scheduler import Scheduler
 from scrapy.utils.python import to_bytes
-from scrapy.utils.job import job_dir
 from scrapy.utils.test import get_crawler
 from tests.spiders import SimpleSpider
 
@@ -29,8 +28,7 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter):
 
     @classmethod
     def from_crawler(cls, crawler):
-        debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
-        df = cls(job_dir(crawler.settings), debug)
+        df = super().from_crawler(crawler)
         df.method = 'from_crawler'
         return df
 
@@ -38,9 +36,8 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter):
 class FromSettingsRFPDupeFilter(RFPDupeFilter):
 
     @classmethod
-    def from_settings(cls, settings):
-        debug = settings.getbool('DUPEFILTER_DEBUG')
-        df = cls(job_dir(settings), debug)
+    def from_settings(cls, settings, *, fingerprinter=None):
+        df = super().from_settings(settings, fingerprinter=fingerprinter)
         df.method = 'from_settings'
         return df
 
@@ -53,7 +50,8 @@ class RFPDupeFilterTest(unittest.TestCase):
 
     def test_df_from_crawler_scheduler(self):
         settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
+                    'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
+                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
@@ -61,14 +59,16 @@ class RFPDupeFilterTest(unittest.TestCase):
 
     def test_df_from_settings_scheduler(self):
         settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter}
+                    'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter,
+                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
         self.assertEqual(scheduler.df.method, 'from_settings')
 
     def test_df_direct_scheduler(self):
-        settings = {'DUPEFILTER_CLASS': DirectDupeFilter}
+        settings = {'DUPEFILTER_CLASS': DirectDupeFilter,
+                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertEqual(scheduler.df.method, 'n/a')
@@ -171,7 +171,8 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_log(self):
         with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': False,
-                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
+                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
+                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             spider = SimpleSpider.from_crawler(crawler)
             dupefilter = _get_dupefilter(crawler=crawler)
@@ -197,7 +198,8 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_log_debug(self):
         with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': True,
-                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
+                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
+                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             spider = SimpleSpider.from_crawler(crawler)
             dupefilter = _get_dupefilter(crawler=crawler)
@@ -230,7 +232,8 @@ class RFPDupeFilterTest(unittest.TestCase):
 
     def test_log_debug_default_dupefilter(self):
         with LogCapture() as log:
-            settings = {'DUPEFILTER_DEBUG': True}
+            settings = {'DUPEFILTER_DEBUG': True,
+                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             spider = SimpleSpider.from_crawler(crawler)
             dupefilter = _get_dupefilter(crawler=crawler)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index fa7d0c8d4..1bd802bcf 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -13,10 +13,11 @@ module with the ``runserver`` argument::
 import os
 import re
 import sys
-import warnings
 from collections import defaultdict
 from urllib.parse import urlparse
+from dataclasses import dataclass
 
+import pytest
 import attr
 from itemadapter import ItemAdapter
 from pydispatch import dispatcher
@@ -50,6 +51,13 @@ class AttrsItem:
     price = attr.ib(default=0)
 
 
+@dataclass
+class DataClassItem:
+    name: str = ""
+    url: str = ""
+    price: int = 0
+
+
 class TestSpider(Spider):
     name = "scrapytest.org"
     allowed_domains = ["scrapytest.org", "localhost"]
@@ -92,17 +100,8 @@ class AttrsItemsSpider(TestSpider):
     item_cls = AttrsItem
 
 
-try:
-    from dataclasses import make_dataclass
-except ImportError:
-    DataClassItemsSpider = None
-else:
-    TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)])
-
-    class DataClassItemsSpider(DictItemsSpider):  # type: ignore[no-redef]
-        def parse_item(self, response):
-            item = super().parse_item(response)
-            return TestDataClass(**item)
+class DataClassItemsSpider(TestSpider):
+    item_cls = DataClassItem
 
 
 class ItemZeroDivisionErrorSpider(TestSpider):
@@ -188,7 +187,7 @@ class CrawlerRun:
         return self.deferred
 
     def stop(self):
-        self.port.stopListening()
+        self.port.stopListening()  # FIXME: wait for this Deferred
         for name, signal in vars(signals).items():
             if not name.startswith('_'):
                 disconnect_all(signal)
@@ -239,79 +238,77 @@ class EngineTest(unittest.TestCase):
     def test_crawler(self):
 
         for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
-            if spider is None:
-                continue
-            self.run = CrawlerRun(spider)
-            yield self.run.run()
-            self._assert_visited_urls()
-            self._assert_scheduled_requests(count=9)
-            self._assert_downloaded_responses(count=9)
-            self._assert_scraped_items()
-            self._assert_signals_caught()
-            self._assert_bytes_received()
+            run = CrawlerRun(spider)
+            yield run.run()
+            self._assert_visited_urls(run)
+            self._assert_scheduled_requests(run, count=9)
+            self._assert_downloaded_responses(run, count=9)
+            self._assert_scraped_items(run)
+            self._assert_signals_caught(run)
+            self._assert_bytes_received(run)
 
     @defer.inlineCallbacks
     def test_crawler_dupefilter(self):
-        self.run = CrawlerRun(TestDupeFilterSpider)
-        yield self.run.run()
-        self._assert_scheduled_requests(count=8)
-        self._assert_dropped_requests()
+        run = CrawlerRun(TestDupeFilterSpider)
+        yield run.run()
+        self._assert_scheduled_requests(run, count=8)
+        self._assert_dropped_requests(run)
 
     @defer.inlineCallbacks
     def test_crawler_itemerror(self):
-        self.run = CrawlerRun(ItemZeroDivisionErrorSpider)
-        yield self.run.run()
-        self._assert_items_error()
+        run = CrawlerRun(ItemZeroDivisionErrorSpider)
+        yield run.run()
+        self._assert_items_error(run)
 
     @defer.inlineCallbacks
     def test_crawler_change_close_reason_on_idle(self):
-        self.run = CrawlerRun(ChangeCloseReasonSpider)
-        yield self.run.run()
-        self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'},
-                         self.run.signals_caught[signals.spider_closed])
+        run = CrawlerRun(ChangeCloseReasonSpider)
+        yield run.run()
+        self.assertEqual({'spider': run.spider, 'reason': 'custom_reason'},
+                         run.signals_caught[signals.spider_closed])
 
-    def _assert_visited_urls(self):
+    def _assert_visited_urls(self, run: CrawlerRun):
         must_be_visited = ["/", "/redirect", "/redirected",
                            "/item1.html", "/item2.html", "/item999.html"]
-        urls_visited = {rp[0].url for rp in self.run.respplug}
-        urls_expected = {self.run.geturl(p) for p in must_be_visited}
+        urls_visited = {rp[0].url for rp in run.respplug}
+        urls_expected = {run.geturl(p) for p in must_be_visited}
         assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"
 
-    def _assert_scheduled_requests(self, count=None):
-        self.assertEqual(count, len(self.run.reqplug))
+    def _assert_scheduled_requests(self, run: CrawlerRun, count=None):
+        self.assertEqual(count, len(run.reqplug))
 
         paths_expected = ['/item999.html', '/item2.html', '/item1.html']
 
-        urls_requested = {rq[0].url for rq in self.run.reqplug}
-        urls_expected = {self.run.geturl(p) for p in paths_expected}
+        urls_requested = {rq[0].url for rq in run.reqplug}
+        urls_expected = {run.geturl(p) for p in paths_expected}
         assert urls_expected <= urls_requested
-        scheduled_requests_count = len(self.run.reqplug)
-        dropped_requests_count = len(self.run.reqdropped)
-        responses_count = len(self.run.respplug)
+        scheduled_requests_count = len(run.reqplug)
+        dropped_requests_count = len(run.reqdropped)
+        responses_count = len(run.respplug)
         self.assertEqual(scheduled_requests_count,
                          dropped_requests_count + responses_count)
-        self.assertEqual(len(self.run.reqreached),
+        self.assertEqual(len(run.reqreached),
                          responses_count)
 
-    def _assert_dropped_requests(self):
-        self.assertEqual(len(self.run.reqdropped), 1)
+    def _assert_dropped_requests(self, run: CrawlerRun):
+        self.assertEqual(len(run.reqdropped), 1)
 
-    def _assert_downloaded_responses(self, count):
+    def _assert_downloaded_responses(self, run: CrawlerRun, count):
         # response tests
-        self.assertEqual(count, len(self.run.respplug))
-        self.assertEqual(count, len(self.run.reqreached))
+        self.assertEqual(count, len(run.respplug))
+        self.assertEqual(count, len(run.reqreached))
 
-        for response, _ in self.run.respplug:
-            if self.run.getpath(response.url) == '/item999.html':
+        for response, _ in run.respplug:
+            if run.getpath(response.url) == '/item999.html':
                 self.assertEqual(404, response.status)
-            if self.run.getpath(response.url) == '/redirect':
+            if run.getpath(response.url) == '/redirect':
                 self.assertEqual(302, response.status)
 
-    def _assert_items_error(self):
-        self.assertEqual(2, len(self.run.itemerror))
-        for item, response, spider, failure in self.run.itemerror:
+    def _assert_items_error(self, run: CrawlerRun):
+        self.assertEqual(2, len(run.itemerror))
+        for item, response, spider, failure in run.itemerror:
             self.assertEqual(failure.value.__class__, ZeroDivisionError)
-            self.assertEqual(spider, self.run.spider)
+            self.assertEqual(spider, run.spider)
 
             self.assertEqual(item['url'], response.url)
             if 'item1.html' in item['url']:
@@ -321,9 +318,9 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('Item 2 name', item['name'])
                 self.assertEqual('200', item['price'])
 
-    def _assert_scraped_items(self):
-        self.assertEqual(2, len(self.run.itemresp))
-        for item, response in self.run.itemresp:
+    def _assert_scraped_items(self, run: CrawlerRun):
+        self.assertEqual(2, len(run.itemresp))
+        for item, response in run.itemresp:
             item = ItemAdapter(item)
             self.assertEqual(item['url'], response.url)
             if 'item1.html' in item['url']:
@@ -333,26 +330,26 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('Item 2 name', item['name'])
                 self.assertEqual('200', item['price'])
 
-    def _assert_headers_received(self):
-        for headers in self.run.headers.values():
+    def _assert_headers_received(self, run: CrawlerRun):
+        for headers in run.headers.values():
             self.assertIn(b"Server", headers)
             self.assertIn(b"TwistedWeb", headers[b"Server"])
             self.assertIn(b"Date", headers)
             self.assertIn(b"Content-Type", headers)
 
-    def _assert_bytes_received(self):
-        self.assertEqual(9, len(self.run.bytes))
-        for request, data in self.run.bytes.items():
+    def _assert_bytes_received(self, run: CrawlerRun):
+        self.assertEqual(9, len(run.bytes))
+        for request, data in run.bytes.items():
             joined_data = b"".join(data)
-            if self.run.getpath(request.url) == "/":
+            if run.getpath(request.url) == "/":
                 self.assertEqual(joined_data, get_testdata("test_site", "index.html"))
-            elif self.run.getpath(request.url) == "/item1.html":
+            elif run.getpath(request.url) == "/item1.html":
                 self.assertEqual(joined_data, get_testdata("test_site", "item1.html"))
-            elif self.run.getpath(request.url) == "/item2.html":
+            elif run.getpath(request.url) == "/item2.html":
                 self.assertEqual(joined_data, get_testdata("test_site", "item2.html"))
-            elif self.run.getpath(request.url) == "/redirected":
+            elif run.getpath(request.url) == "/redirected":
                 self.assertEqual(joined_data, b"Redirected here")
-            elif self.run.getpath(request.url) == '/redirect':
+            elif run.getpath(request.url) == '/redirect':
                 self.assertEqual(
                     joined_data,
                     b"\n<html>\n"
@@ -364,7 +361,7 @@ class EngineTest(unittest.TestCase):
                     b"    </body>\n"
                     b"</html>\n"
                 )
-            elif self.run.getpath(request.url) == "/tem999.html":
+            elif run.getpath(request.url) == "/tem999.html":
                 self.assertEqual(
                     joined_data,
                     b"\n<html>\n"
@@ -375,27 +372,27 @@ class EngineTest(unittest.TestCase):
                     b"  </body>\n"
                     b"</html>\n"
                 )
-            elif self.run.getpath(request.url) == "/numbers":
+            elif run.getpath(request.url) == "/numbers":
                 # signal was fired multiple times
                 self.assertTrue(len(data) > 1)
                 # bytes were received in order
                 numbers = [str(x).encode("utf8") for x in range(2**18)]
                 self.assertEqual(joined_data, b"".join(numbers))
 
-    def _assert_signals_caught(self):
-        assert signals.engine_started in self.run.signals_caught
-        assert signals.engine_stopped in self.run.signals_caught
-        assert signals.spider_opened in self.run.signals_caught
-        assert signals.spider_idle in self.run.signals_caught
-        assert signals.spider_closed in self.run.signals_caught
-        assert signals.headers_received in self.run.signals_caught
+    def _assert_signals_caught(self, run: CrawlerRun):
+        assert signals.engine_started in run.signals_caught
+        assert signals.engine_stopped in run.signals_caught
+        assert signals.spider_opened in run.signals_caught
+        assert signals.spider_idle in run.signals_caught
+        assert signals.spider_closed in run.signals_caught
+        assert signals.headers_received in run.signals_caught
 
-        self.assertEqual({'spider': self.run.spider},
-                         self.run.signals_caught[signals.spider_opened])
-        self.assertEqual({'spider': self.run.spider},
-                         self.run.signals_caught[signals.spider_idle])
-        self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
-                         self.run.signals_caught[signals.spider_closed])
+        self.assertEqual({'spider': run.spider},
+                         run.signals_caught[signals.spider_opened])
+        self.assertEqual({'spider': run.spider},
+                         run.signals_caught[signals.spider_idle])
+        self.assertEqual({'spider': run.spider, 'reason': 'finished'},
+                         run.signals_caught[signals.spider_closed])
 
     @defer.inlineCallbacks
     def test_close_downloader(self):
@@ -407,28 +404,29 @@ class EngineTest(unittest.TestCase):
         e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
         yield e.open_spider(TestSpider(), [])
         e.start()
-        yield self.assertFailure(e.start(), RuntimeError).addBoth(
-            lambda exc: self.assertEqual(str(exc), "Engine already running")
-        )
-        yield e.stop()
+        try:
+            yield self.assertFailure(e.start(), RuntimeError).addBoth(
+                lambda exc: self.assertEqual(str(exc), "Engine already running")
+            )
+        finally:
+            yield e.stop()
 
     @defer.inlineCallbacks
     def test_close_spiders_downloader(self):
-        with warnings.catch_warnings(record=True) as warning_list:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="ExecutionEngine.open_spiders is deprecated, "
+                                "please use ExecutionEngine.spider instead"):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             yield e.open_spider(TestSpider(), [])
             self.assertEqual(len(e.open_spiders), 1)
             yield e.close()
             self.assertEqual(len(e.open_spiders), 0)
-            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(
-                str(warning_list[0].message),
-                "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
-            )
 
     @defer.inlineCallbacks
     def test_close_engine_spiders_downloader(self):
-        with warnings.catch_warnings(record=True) as warning_list:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="ExecutionEngine.open_spiders is deprecated, "
+                                "please use ExecutionEngine.spider instead"):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             yield e.open_spider(TestSpider(), [])
             e.start()
@@ -436,61 +434,47 @@ class EngineTest(unittest.TestCase):
             yield e.close()
             self.assertFalse(e.running)
             self.assertEqual(len(e.open_spiders), 0)
-            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(
-                str(warning_list[0].message),
-                "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
-            )
 
     @defer.inlineCallbacks
     def test_crawl_deprecated_spider_arg(self):
-        with warnings.catch_warnings(record=True) as warning_list:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="Passing a 'spider' argument to "
+                                "ExecutionEngine.crawl is deprecated"):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             spider = TestSpider()
             yield e.open_spider(spider, [])
             e.start()
             e.crawl(Request("data:,"), spider)
             yield e.close()
-            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(
-                str(warning_list[0].message),
-                "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated",
-            )
 
     @defer.inlineCallbacks
     def test_download_deprecated_spider_arg(self):
-        with warnings.catch_warnings(record=True) as warning_list:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="Passing a 'spider' argument to "
+                                "ExecutionEngine.download is deprecated"):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             spider = TestSpider()
             yield e.open_spider(spider, [])
             e.start()
             e.download(Request("data:,"), spider)
             yield e.close()
-            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(
-                str(warning_list[0].message),
-                "Passing a 'spider' argument to ExecutionEngine.download is deprecated",
-            )
 
     @defer.inlineCallbacks
     def test_deprecated_schedule(self):
-        with warnings.catch_warnings(record=True) as warning_list:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="ExecutionEngine.schedule is deprecated, please use "
+                                "ExecutionEngine.crawl or ExecutionEngine.download instead"):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             spider = TestSpider()
             yield e.open_spider(spider, [])
             e.start()
             e.schedule(Request("data:,"), spider)
             yield e.close()
-            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(
-                str(warning_list[0].message),
-                "ExecutionEngine.schedule is deprecated, please use "
-                "ExecutionEngine.crawl or ExecutionEngine.download instead",
-            )
 
     @defer.inlineCallbacks
     def test_deprecated_has_capacity(self):
-        with warnings.catch_warnings(record=True) as warning_list:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="ExecutionEngine.has_capacity is deprecated"):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             self.assertTrue(e.has_capacity())
             spider = TestSpider()
@@ -499,8 +483,6 @@ class EngineTest(unittest.TestCase):
             e.start()
             yield e.close()
             self.assertTrue(e.has_capacity())
-            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated")
 
 
 if __name__ == "__main__":
diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py
index 0ba69e096..933e4067d 100644
--- a/tests/test_engine_stop_download_bytes.py
+++ b/tests/test_engine_stop_download_bytes.py
@@ -23,36 +23,34 @@ class BytesReceivedEngineTest(EngineTest):
     @defer.inlineCallbacks
     def test_crawler(self):
         for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
-            if spider is None:
-                continue
-            self.run = BytesReceivedCrawlerRun(spider)
+            run = BytesReceivedCrawlerRun(spider)
             with LogCapture() as log:
-                yield self.run.run()
+                yield run.run()
                 log.check_present(("scrapy.core.downloader.handlers.http11",
                                    "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> "
+                                   f"Download stopped for <GET http://localhost:{run.portno}/redirected> "
                                    "from signal handler BytesReceivedCrawlerRun.bytes_received"))
                 log.check_present(("scrapy.core.downloader.handlers.http11",
                                    "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/> "
+                                   f"Download stopped for <GET http://localhost:{run.portno}/> "
                                    "from signal handler BytesReceivedCrawlerRun.bytes_received"))
                 log.check_present(("scrapy.core.downloader.handlers.http11",
                                    "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> "
+                                   f"Download stopped for <GET http://localhost:{run.portno}/numbers> "
                                    "from signal handler BytesReceivedCrawlerRun.bytes_received"))
-            self._assert_visited_urls()
-            self._assert_scheduled_requests(count=9)
-            self._assert_downloaded_responses(count=9)
-            self._assert_signals_caught()
-            self._assert_headers_received()
-            self._assert_bytes_received()
+            self._assert_visited_urls(run)
+            self._assert_scheduled_requests(run, count=9)
+            self._assert_downloaded_responses(run, count=9)
+            self._assert_signals_caught(run)
+            self._assert_headers_received(run)
+            self._assert_bytes_received(run)
 
-    def _assert_bytes_received(self):
-        self.assertEqual(9, len(self.run.bytes))
-        for request, data in self.run.bytes.items():
+    def _assert_bytes_received(self, run: CrawlerRun):
+        self.assertEqual(9, len(run.bytes))
+        for request, data in run.bytes.items():
             joined_data = b"".join(data)
             self.assertTrue(len(data) == 1)  # signal was fired only once
-            if self.run.getpath(request.url) == "/numbers":
+            if run.getpath(request.url) == "/numbers":
                 # Received bytes are not the complete response. The exact amount depends
                 # on the buffer size, which can vary, so we only check that the amount
                 # of received bytes is strictly less than the full response.
diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py
index fad6643ad..8975d0e3f 100644
--- a/tests/test_engine_stop_download_headers.py
+++ b/tests/test_engine_stop_download_headers.py
@@ -23,34 +23,32 @@ class HeadersReceivedEngineTest(EngineTest):
     @defer.inlineCallbacks
     def test_crawler(self):
         for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
-            if spider is None:
-                continue
-            self.run = HeadersReceivedCrawlerRun(spider)
+            run = HeadersReceivedCrawlerRun(spider)
             with LogCapture() as log:
-                yield self.run.run()
+                yield run.run()
                 log.check_present(("scrapy.core.downloader.handlers.http11",
                                    "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> from"
+                                   f"Download stopped for <GET http://localhost:{run.portno}/redirected> from"
                                    " signal handler HeadersReceivedCrawlerRun.headers_received"))
                 log.check_present(("scrapy.core.downloader.handlers.http11",
                                    "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/> from signal"
+                                   f"Download stopped for <GET http://localhost:{run.portno}/> from signal"
                                    " handler HeadersReceivedCrawlerRun.headers_received"))
                 log.check_present(("scrapy.core.downloader.handlers.http11",
                                    "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> from"
+                                   f"Download stopped for <GET http://localhost:{run.portno}/numbers> from"
                                    " signal handler HeadersReceivedCrawlerRun.headers_received"))
-            self._assert_visited_urls()
-            self._assert_downloaded_responses(count=6)
-            self._assert_signals_caught()
-            self._assert_bytes_received()
-            self._assert_headers_received()
+            self._assert_visited_urls(run)
+            self._assert_downloaded_responses(run, count=6)
+            self._assert_signals_caught(run)
+            self._assert_bytes_received(run)
+            self._assert_headers_received(run)
 
-    def _assert_bytes_received(self):
-        self.assertEqual(0, len(self.run.bytes))
+    def _assert_bytes_received(self, run: CrawlerRun):
+        self.assertEqual(0, len(run.bytes))
 
-    def _assert_visited_urls(self):
+    def _assert_visited_urls(self, run: CrawlerRun):
         must_be_visited = ["/", "/redirect", "/redirected"]
-        urls_visited = {rp[0].url for rp in self.run.respplug}
-        urls_expected = {self.run.geturl(p) for p in must_be_visited}
+        urls_visited = {rp[0].url for rp in run.respplug}
+        urls_expected = {run.geturl(p) for p in must_be_visited}
         assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index ec48f8d4a..ecd1b59d3 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -22,6 +22,7 @@ from urllib.parse import urljoin, quote
 from urllib.request import pathname2url
 
 import lxml.etree
+import pytest
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial import unittest
@@ -30,7 +31,6 @@ from zope.interface import implementer
 from zope.interface.verify import verifyObject
 
 import scrapy
-from scrapy.crawler import CrawlerRunner
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.exporters import CsvItemExporter
 from scrapy.extensions.feedexport import (
@@ -697,9 +697,9 @@ class FeedExportTest(FeedExportTestBase):
         content = {}
         try:
             with MockServer() as s:
-                runner = CrawlerRunner(Settings(settings))
                 spider_cls.start_urls = [s.url('/')]
-                yield runner.crawl(spider_cls)
+                crawler = get_crawler(spider_cls, settings)
+                yield crawler.crawl()
 
             for file_path, feed_options in FEEDS.items():
                 if not os.path.exists(str(file_path)):
@@ -1554,9 +1554,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
         content = {}
         try:
             with MockServer() as s:
-                runner = CrawlerRunner(Settings(settings))
                 spider_cls.start_urls = [s.url('/')]
-                yield runner.crawl(spider_cls)
+                crawler = get_crawler(spider_cls, settings)
+                yield crawler.crawl()
 
             for file_path, feed_options in FEEDS.items():
                 if not os.path.exists(str(file_path)):
@@ -2026,9 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
         content = defaultdict(list)
         try:
             with MockServer() as s:
-                runner = CrawlerRunner(Settings(settings))
                 spider_cls.start_urls = [s.url('/')]
-                yield runner.crawl(spider_cls)
+                crawler = get_crawler(spider_cls, settings)
+                yield crawler.crawl()
 
             for path, feed in FEEDS.items():
                 dir_name = os.path.dirname(path)
@@ -2048,7 +2048,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['jl']:
@@ -2064,7 +2064,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
             },
         })
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         data = yield self.exported_data(items, settings)
         for batch in data['csv']:
             got_batch = csv.DictReader(to_unicode(batch).splitlines())
@@ -2080,7 +2080,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
             },
         })
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['xml']:
@@ -2098,7 +2098,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
             },
         })
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         # XML
@@ -2123,7 +2123,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
             },
         })
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import pickle
@@ -2140,7 +2140,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
             },
         })
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import marshal
@@ -2166,7 +2166,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             'FEED_EXPORT_BATCH_ITEM_COUNT': 2
         }
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows, settings=Settings(settings))
+        yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
         """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """
@@ -2382,9 +2382,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     yield item
 
         with MockServer() as server:
-            runner = CrawlerRunner(Settings(settings))
             TestSpider.start_urls = [server.url('/')]
-            yield runner.crawl(TestSpider)
+            crawler = get_crawler(TestSpider, settings)
+            yield crawler.crawl()
 
         self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1)
         for stub in CustomS3FeedStorage.stubs[:-1]:
@@ -2434,25 +2434,16 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):
                 'file': StdoutFeedStorageWithoutFeedOptions
             },
         }
-        crawler = get_crawler(settings_dict=settings_dict)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
         spider = scrapy.Spider("default")
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="StdoutFeedStorageWithoutFeedOptions does not support "
+                                "the 'feed_options' keyword argument."):
             feed_exporter.open_spider(spider)
-            messages = tuple(str(item.message) for item in w
-                             if item.category is ScrapyDeprecationWarning)
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        "StdoutFeedStorageWithoutFeedOptions does not support "
-                        "the 'feed_options' keyword argument. Add a "
-                        "'feed_options' parameter to its signature to remove "
-                        "this warning. This parameter will become mandatory "
-                        "in a future version of Scrapy."
-                    ),
-                )
-            )
 
 
 class FileFeedStorageWithoutFeedOptions(FileFeedStorage):
@@ -2476,25 +2467,16 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
                     'file': FileFeedStorageWithoutFeedOptions
                 },
             }
-            crawler = get_crawler(settings_dict=settings_dict)
-            feed_exporter = FeedExporter.from_crawler(crawler)
+            with pytest.warns(ScrapyDeprecationWarning,
+                              match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+                crawler = get_crawler(settings_dict=settings_dict)
+                feed_exporter = FeedExporter.from_crawler(crawler)
         spider = scrapy.Spider("default")
-        with warnings.catch_warnings(record=True) as w:
+
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="FileFeedStorageWithoutFeedOptions does not support "
+                                "the 'feed_options' keyword argument."):
             feed_exporter.open_spider(spider)
-            messages = tuple(str(item.message) for item in w
-                             if item.category is ScrapyDeprecationWarning)
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        "FileFeedStorageWithoutFeedOptions does not support "
-                        "the 'feed_options' keyword argument. Add a "
-                        "'feed_options' parameter to its signature to remove "
-                        "this warning. This parameter will become mandatory "
-                        "in a future version of Scrapy."
-                    ),
-                )
-            )
 
 
 class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
@@ -2524,26 +2506,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
                 'file': S3FeedStorageWithoutFeedOptions
             },
         }
-        crawler = get_crawler(settings_dict=settings_dict)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
         spider = scrapy.Spider("default")
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="S3FeedStorageWithoutFeedOptions does not support "
+                                "the 'feed_options' keyword argument."):
             feed_exporter.open_spider(spider)
-            messages = tuple(str(item.message) for item in w
-                             if item.category is ScrapyDeprecationWarning)
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        "S3FeedStorageWithoutFeedOptions does not support "
-                        "the 'feed_options' keyword argument. Add a "
-                        "'feed_options' parameter to its signature to remove "
-                        "this warning. This parameter will become mandatory "
-                        "in a future version of Scrapy."
-                    ),
-                )
-            )
 
     def test_from_crawler(self):
         settings_dict = {
@@ -2552,26 +2526,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
                 'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler
             },
         }
-        crawler = get_crawler(settings_dict=settings_dict)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
         spider = scrapy.Spider("default")
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
+                                "the 'feed_options' keyword argument."):
             feed_exporter.open_spider(spider)
-            messages = tuple(str(item.message) for item in w
-                             if item.category is ScrapyDeprecationWarning)
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
-                        "does not support the 'feed_options' keyword argument. Add a "
-                        "'feed_options' parameter to its signature to remove "
-                        "this warning. This parameter will become mandatory "
-                        "in a future version of Scrapy."
-                    ),
-                )
-            )
 
 
 class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):
@@ -2601,26 +2567,18 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
                 'file': FTPFeedStorageWithoutFeedOptions
             },
         }
-        crawler = get_crawler(settings_dict=settings_dict)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
         spider = scrapy.Spider("default")
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="FTPFeedStorageWithoutFeedOptions does not support "
+                                "the 'feed_options' keyword argument."):
             feed_exporter.open_spider(spider)
-            messages = tuple(str(item.message) for item in w
-                             if item.category is ScrapyDeprecationWarning)
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        "FTPFeedStorageWithoutFeedOptions does not support "
-                        "the 'feed_options' keyword argument. Add a "
-                        "'feed_options' parameter to its signature to remove "
-                        "this warning. This parameter will become mandatory "
-                        "in a future version of Scrapy."
-                    ),
-                )
-            )
 
     def test_from_crawler(self):
         settings_dict = {
@@ -2629,50 +2587,50 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
                 'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler
             },
         }
-        crawler = get_crawler(settings_dict=settings_dict)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
         spider = scrapy.Spider("default")
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
+                                "the 'feed_options' keyword argument."):
             feed_exporter.open_spider(spider)
-            messages = tuple(str(item.message) for item in w
-                             if item.category is ScrapyDeprecationWarning)
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
-                        "does not support the 'feed_options' keyword argument. Add a "
-                        "'feed_options' parameter to its signature to remove "
-                        "this warning. This parameter will become mandatory "
-                        "in a future version of Scrapy."
-                    ),
-                )
-            )
 
 
 class URIParamsTest:
 
     spider_name = "uri_params_spider"
+    deprecated_options = False
 
     def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
         raise NotImplementedError
 
+    def _crawler_feed_exporter(self, settings):
+        if self.deprecated_options:
+            with pytest.warns(ScrapyDeprecationWarning,
+                              match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+                crawler = get_crawler(settings_dict=settings)
+                feed_exporter = FeedExporter.from_crawler(crawler)
+        else:
+            crawler = get_crawler(settings_dict=settings)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+        return crawler, feed_exporter
+
     def test_default(self):
         settings = self.build_settings(
             uri='file:///tmp/%(name)s',
         )
-        crawler = get_crawler(settings_dict=settings)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        crawler, feed_exporter = self._crawler_feed_exporter(settings)
         spider = scrapy.Spider(self.spider_name)
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", ScrapyDeprecationWarning)
             feed_exporter.open_spider(spider)
-            messages = tuple(
-                str(item.message) for item in w
-                if item.category is ScrapyDeprecationWarning
-            )
-            self.assertEqual(messages, tuple())
 
         self.assertEqual(
             feed_exporter.slots[0].uri,
@@ -2687,28 +2645,13 @@ class URIParamsTest:
             uri='file:///tmp/%(name)s',
             uri_params=uri_params,
         )
-        crawler = get_crawler(settings_dict=settings)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        crawler, feed_exporter = self._crawler_feed_exporter(settings)
         spider = scrapy.Spider(self.spider_name)
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="Modifying the params dictionary in-place"):
             feed_exporter.open_spider(spider)
-            messages = tuple(
-                str(item.message) for item in w
-                if item.category is ScrapyDeprecationWarning
-            )
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        'Modifying the params dictionary in-place in the '
-                        'function defined in the FEED_URI_PARAMS setting or '
-                        'in the uri_params key of the FEEDS setting is '
-                        'deprecated. The function must return a new '
-                        'dictionary instead.'
-                    ),
-                )
-            )
 
         self.assertEqual(
             feed_exporter.slots[0].uri,
@@ -2723,18 +2666,14 @@ class URIParamsTest:
             uri='file:///tmp/%(name)s',
             uri_params=uri_params,
         )
-        crawler = get_crawler(settings_dict=settings)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        crawler, feed_exporter = self._crawler_feed_exporter(settings)
         spider = scrapy.Spider(self.spider_name)
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", ScrapyDeprecationWarning)
             with self.assertRaises(KeyError):
                 feed_exporter.open_spider(spider)
-            messages = tuple(
-                str(item.message) for item in w
-                if item.category is ScrapyDeprecationWarning
-            )
-            self.assertEqual(messages, tuple())
 
     def test_params_as_is(self):
         def uri_params(params, spider):
@@ -2744,17 +2683,12 @@ class URIParamsTest:
             uri='file:///tmp/%(name)s',
             uri_params=uri_params,
         )
-        crawler = get_crawler(settings_dict=settings)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        crawler, feed_exporter = self._crawler_feed_exporter(settings)
         spider = scrapy.Spider(self.spider_name)
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", ScrapyDeprecationWarning)
             feed_exporter.open_spider(spider)
-            messages = tuple(
-                str(item.message) for item in w
-                if item.category is ScrapyDeprecationWarning
-            )
-            self.assertEqual(messages, tuple())
 
         self.assertEqual(
             feed_exporter.slots[0].uri,
@@ -2769,17 +2703,12 @@ class URIParamsTest:
             uri='file:///tmp/%(foo)s',
             uri_params=uri_params,
         )
-        crawler = get_crawler(settings_dict=settings)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        crawler, feed_exporter = self._crawler_feed_exporter(settings)
         spider = scrapy.Spider(self.spider_name)
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", ScrapyDeprecationWarning)
             feed_exporter.open_spider(spider)
-            messages = tuple(
-                str(item.message) for item in w
-                if item.category is ScrapyDeprecationWarning
-            )
-            self.assertEqual(messages, tuple())
 
         self.assertEqual(
             feed_exporter.slots[0].uri,
@@ -2788,6 +2717,7 @@ class URIParamsTest:
 
 
 class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
+    deprecated_options = True
 
     def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
         extra_settings = {}
@@ -2800,6 +2730,7 @@ class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
 
 
 class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase):
+    deprecated_options = False
 
     def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
         options = {
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 6381f895b..f3bb23bda 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -5,8 +5,8 @@ from twisted.internet import defer
 from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase as TwistedTestCase
 
-from scrapy.crawler import CrawlerRunner
 from scrapy.exceptions import DropItem
+from scrapy.utils.test import get_crawler
 from scrapy.http import Request, Response
 from scrapy.item import Item, Field
 from scrapy.logformatter import LogFormatter
@@ -202,7 +202,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
 
     @defer.inlineCallbacks
     def test_show_messages(self):
-        crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider)
+        crawler = get_crawler(ItemSpider, self.base_settings)
         with LogCapture() as lc:
             yield crawler.crawl(mockserver=self.mockserver)
         self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc))
@@ -213,7 +213,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
     def test_skip_messages(self):
         settings = self.base_settings.copy()
         settings['LOG_FORMATTER'] = SkipMessagesLogFormatter
-        crawler = CrawlerRunner(settings).create_crawler(ItemSpider)
+        crawler = get_crawler(ItemSpider, settings)
         with LogCapture() as lc:
             yield crawler.crawl(mockserver=self.mockserver)
         self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc))
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index f49fda701..e46532a1c 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -64,6 +64,7 @@ class FileDownloadCrawlTestCase(TestCase):
         self.tmpmediastore = self.mktemp()
         os.mkdir(self.tmpmediastore)
         self.settings = {
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
             'ITEM_PIPELINES': {self.pipeline_class: 1},
             self.store_setting_key: self.tmpmediastore,
         }
@@ -78,8 +79,10 @@ class FileDownloadCrawlTestCase(TestCase):
     def _on_item_scraped(self, item):
         self.items.append(item)
 
-    def _create_crawler(self, spider_class, **kwargs):
-        crawler = self.runner.create_crawler(spider_class, **kwargs)
+    def _create_crawler(self, spider_class, runner=None, **kwargs):
+        if runner is None:
+            runner = self.runner
+        crawler = runner.create_crawler(spider_class, **kwargs)
         crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
         return crawler
 
@@ -167,9 +170,8 @@ class FileDownloadCrawlTestCase(TestCase):
     def test_download_media_redirected_allowed(self):
         settings = dict(self.settings)
         settings.update({'MEDIA_ALLOW_REDIRECTS': True})
-        self.runner = CrawlerRunner(settings)
-
-        crawler = self._create_crawler(RedirectedMediaDownloadSpider)
+        runner = CrawlerRunner(settings)
+        crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner)
         with LogCapture() as log:
             yield crawler.crawl(
                 self.mockserver.url("/files/images/"),
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index 25d9657d5..0406d906f 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -2,8 +2,8 @@ from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
 from scrapy import Request, signals
-from scrapy.crawler import CrawlerRunner
 from scrapy.http.response import Response
+from scrapy.utils.test import get_crawler
 
 from testfixtures import LogCapture
 
@@ -71,7 +71,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_response_200(self):
         url = self.mockserver.url("/status?n=200")
-        crawler = CrawlerRunner().create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         response = crawler.spider.meta["responses"][0]
         self.assertEqual(response.request.url, url)
@@ -80,7 +80,7 @@ class CrawlTestCase(TestCase):
     def test_response_error(self):
         for status in ("404", "500"):
             url = self.mockserver.url(f"/status?n={status}")
-            crawler = CrawlerRunner().create_crawler(SingleRequestSpider)
+            crawler = get_crawler(SingleRequestSpider)
             yield crawler.crawl(seed=url, mockserver=self.mockserver)
             failure = crawler.spider.meta["failure"]
             response = failure.value.response
@@ -90,12 +90,11 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_downloader_middleware_raise_exception(self):
         url = self.mockserver.url("/status?n=200")
-        runner = CrawlerRunner(settings={
+        crawler = get_crawler(SingleRequestSpider, {
             "DOWNLOADER_MIDDLEWARES": {
                 RaiseExceptionRequestMiddleware: 590,
             },
         })
-        crawler = runner.create_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         failure = crawler.spider.meta["failure"]
         self.assertEqual(failure.request.url, url)
@@ -117,12 +116,11 @@ class CrawlTestCase(TestCase):
             signal_params["request"] = request
 
         url = self.mockserver.url("/status?n=200")
-        runner = CrawlerRunner(settings={
+        crawler = get_crawler(SingleRequestSpider, {
             "DOWNLOADER_MIDDLEWARES": {
                 ProcessResponseMiddleware: 595,
             }
         })
-        crawler = runner.create_crawler(SingleRequestSpider)
         crawler.signals.connect(signal_handler, signal=signals.response_received)
 
         with LogCapture() as log:
@@ -147,13 +145,12 @@ class CrawlTestCase(TestCase):
         The spider callback should receive the overridden response.request
         """
         url = self.mockserver.url("/status?n=200")
-        runner = CrawlerRunner(settings={
+        crawler = get_crawler(SingleRequestSpider, {
             "DOWNLOADER_MIDDLEWARES": {
                 RaiseExceptionRequestMiddleware: 590,
                 CatchExceptionOverrideRequestMiddleware: 595,
             },
         })
-        crawler = runner.create_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         response = crawler.spider.meta["responses"][0]
         self.assertEqual(response.body, b"Caught ZeroDivisionError")
@@ -168,13 +165,12 @@ class CrawlTestCase(TestCase):
         The spider callback should receive the original response.request
         """
         url = self.mockserver.url("/status?n=200")
-        runner = CrawlerRunner(settings={
+        crawler = get_crawler(SingleRequestSpider, {
             "DOWNLOADER_MIDDLEWARES": {
                 RaiseExceptionRequestMiddleware: 590,
                 CatchExceptionDoNotOverrideRequestMiddleware: 595,
             },
         })
-        crawler = runner.create_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         response = crawler.spider.meta["responses"][0]
         self.assertEqual(response.body, b"Caught ZeroDivisionError")
@@ -186,12 +182,11 @@ class CrawlTestCase(TestCase):
         Downloader middleware which returns a response with a
         specific 'request' attribute, with an alternative callback
         """
-        runner = CrawlerRunner(settings={
+        crawler = get_crawler(AlternativeCallbacksSpider, {
             "DOWNLOADER_MIDDLEWARES": {
                 AlternativeCallbacksMiddleware: 595,
             }
         })
-        crawler = runner.create_crawler(AlternativeCallbacksSpider)
 
         with LogCapture() as log:
             url = self.mockserver.url("/status?n=200")
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 473a93e69..002a04358 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -3,7 +3,7 @@ from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
 from scrapy.http import Request
-from scrapy.crawler import CrawlerRunner
+from scrapy.utils.test import get_crawler
 from tests.spiders import MockServerSpider
 from tests.mockserver import MockServer
 
@@ -140,14 +140,13 @@ class CallbackKeywordArgumentsTestCase(TestCase):
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
-        self.runner = CrawlerRunner()
 
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
 
     @defer.inlineCallbacks
     def test_callback_kwargs(self):
-        crawler = self.runner.create_crawler(KeywordArgumentsSpider)
+        crawler = get_crawler(KeywordArgumentsSpider)
         with LogCapture() as log:
             yield crawler.crawl(mockserver=self.mockserver)
         self.assertTrue(all(crawler.spider.checks))
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 2d4bfa165..ac66056ba 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -52,6 +52,7 @@ class MockCrawler(Crawler):
             SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
             JOBDIR=jobdir,
             DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
+            REQUEST_FINGERPRINTER_IMPLEMENTATION='VERSION',
         )
         super().__init__(Spider, settings)
         self.engine = MockEngine(downloader=MockDownloader())
@@ -334,7 +335,7 @@ class TestIncompatibility(unittest.TestCase):
             SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
             CONCURRENT_REQUESTS_PER_IP=1,
         )
-        crawler = Crawler(Spider, settings)
+        crawler = get_crawler(Spider, settings)
         scheduler = Scheduler.from_crawler(crawler)
         spider = Spider(name='spider')
         scheduler.open(spider)
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index bf90b4320..fc234a83d 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -7,10 +7,10 @@ from twisted.internet import defer
 from twisted.trial.unittest import TestCase as TwistedTestCase
 
 from scrapy.core.scheduler import BaseScheduler
-from scrapy.crawler import CrawlerRunner
 from scrapy.http import Request
 from scrapy.spiders import Spider
-from scrapy.utils.request import request_fingerprint
+from scrapy.utils.request import fingerprint
+from scrapy.utils.test import get_crawler
 
 from tests.mockserver import MockServer
 
@@ -21,13 +21,13 @@ URLS = [urljoin("https://example.org", p) for p in PATHS]
 
 class MinimalScheduler:
     def __init__(self) -> None:
-        self.requests: Dict[str, Request] = {}
+        self.requests: Dict[bytes, Request] = {}
 
     def has_pending_requests(self) -> bool:
         return bool(self.requests)
 
     def enqueue_request(self, request: Request) -> bool:
-        fp = request_fingerprint(request)
+        fp = fingerprint(request)
         if fp not in self.requests:
             self.requests[fp] = request
             return True
@@ -147,9 +147,12 @@ class MinimalSchedulerCrawlTest(TwistedTestCase):
     @defer.inlineCallbacks
     def test_crawl(self):
         with MockServer() as mockserver:
-            settings = {"SCHEDULER": self.scheduler_cls}
+            settings = {
+                "SCHEDULER": self.scheduler_cls,
+            }
             with LogCapture() as log:
-                yield CrawlerRunner(settings).crawl(TestSpider, mockserver)
+                crawler = get_crawler(TestSpider, settings)
+                yield crawler.crawl(mockserver)
             for path in PATHS:
                 self.assertIn(f"{{'path': '{path}'}}", str(log))
             self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log))
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 8a35e9fd7..3719c7c9f 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -96,7 +96,10 @@ class SpiderLoaderTest(unittest.TestCase):
 
     def test_crawler_runner_loading(self):
         module = 'tests.test_spiderloader.test_spiders.spider1'
-        runner = CrawlerRunner({'SPIDER_MODULES': [module]})
+        runner = CrawlerRunner({
+            'SPIDER_MODULES': [module],
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
+        })
 
         self.assertRaisesRegex(KeyError, 'Spider not found',
                                runner.create_crawler, 'spider2')
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 1ef4eeb14..46452415a 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -3,6 +3,7 @@ import os
 import tempfile
 import shutil
 import contextlib
+import warnings
 
 from pytest import warns
 
@@ -68,20 +69,21 @@ class GetProjectSettingsTestCase(unittest.TestCase):
         envvars = {
             'SCRAPY_SETTINGS_MODULE': value,
         }
-        with set_env(**envvars), warns(None) as warnings:
-            settings = get_project_settings()
-        assert not warnings
+        with warnings.catch_warnings():
+            warnings.simplefilter("error")
+            with set_env(**envvars):
+                settings = get_project_settings()
+
         assert settings.get('SETTINGS_MODULE') == value
 
     def test_invalid_envvar(self):
         envvars = {
             'SCRAPY_FOO': 'bar',
         }
-        with set_env(**envvars), warns(None) as warnings:
-            get_project_settings()
-        assert len(warnings) == 1
-        assert warnings[0].category == ScrapyDeprecationWarning
-        assert str(warnings[0].message).endswith(': FOO')
+        with warns(ScrapyDeprecationWarning, match=': FOO') as record:
+            with set_env(**envvars):
+                get_project_settings()
+        assert len(record) == 1
 
     def test_valid_and_invalid_envvars(self):
         value = 'tests.test_cmdline.settings'
@@ -89,9 +91,8 @@ class GetProjectSettingsTestCase(unittest.TestCase):
             'SCRAPY_FOO': 'bar',
             'SCRAPY_SETTINGS_MODULE': value,
         }
-        with set_env(**envvars), warns(None) as warnings:
-            settings = get_project_settings()
-        assert len(warnings) == 1
-        assert warnings[0].category == ScrapyDeprecationWarning
-        assert str(warnings[0].message).endswith(': FOO')
+        with warns(ScrapyDeprecationWarning, match=': FOO') as record:
+            with set_env(**envvars):
+                settings = get_project_settings()
+        assert len(record) == 1
         assert settings.get('SETTINGS_MODULE') == value
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index e9edfee98..5ee772c0b 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -308,13 +308,22 @@ class RequestFingerprintTest(FingerprintTest):
         ),
     )
 
+    def setUp(self) -> None:
+        warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+
+    def tearDown(self) -> None:
+        warnings.simplefilter("default", ScrapyDeprecationWarning)
+
     @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
     def test_part_separation(self):
         super().test_part_separation()
 
+
+class RequestFingerprintDeprecationTest(unittest.TestCase):
+
     def test_deprecation_default_parameters(self):
         with pytest.warns(ScrapyDeprecationWarning) as warnings:
-            self.function(Request("http://www.example.com"))
+            request_fingerprint(Request("http://www.example.com"))
         messages = [str(warning.message) for warning in warnings]
         self.assertTrue(
             any(
@@ -326,7 +335,7 @@ class RequestFingerprintTest(FingerprintTest):
 
     def test_deprecation_non_default_parameters(self):
         with pytest.warns(ScrapyDeprecationWarning) as warnings:
-            self.function(Request("http://www.example.com"), keep_fragments=True)
+            request_fingerprint(Request("http://www.example.com"), keep_fragments=True)
         messages = [str(warning.message) for warning in warnings]
         self.assertTrue(
             any(
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 0a09f6109..d20852e62 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,7 +1,9 @@
 import os
 import unittest
+import warnings
 from urllib.parse import urlparse
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse, HtmlResponse
 from scrapy.utils.python import to_bytes
 from scrapy.utils.response import (response_httprepr, open_in_browser,
@@ -15,14 +17,21 @@ class ResponseUtilsTest(unittest.TestCase):
     dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response')
 
     def test_response_httprepr(self):
-        r1 = Response("http://www.example.com")
-        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
 
-        r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body=b"Some body")
-        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
+            r1 = Response("http://www.example.com")
+            self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')
 
-        r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body=b"Some body")
-        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')
+            r1 = Response("http://www.example.com", status=404,
+                          headers={"Content-type": "text/html"}, body=b"Some body")
+            self.assertEqual(response_httprepr(r1),
+                             b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
+
+            r1 = Response("http://www.example.com", status=6666,
+                          headers={"Content-type": "text/html"}, body=b"Some body")
+            self.assertEqual(response_httprepr(r1),
+                             b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')
 
     def test_open_in_browser(self):
         url = "http:///www.example.com/some/page.html"

From f60c7ae768fa2f238ea6d7646098fe99ef8ad461 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 20 Jul 2022 14:01:22 +0500
Subject: [PATCH 633/716] Fixed heading levels in downloader middleware docs
 (#5567)

---
 docs/topics/downloader-middleware.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 29e350651..986da0476 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -955,7 +955,7 @@ default because HTTP specs say so.
 .. setting:: RETRY_PRIORITY_ADJUST
 
 RETRY_PRIORITY_ADJUST
----------------------
+^^^^^^^^^^^^^^^^^^^^^
 
 Default: ``-1``
 
@@ -1119,7 +1119,7 @@ In order to use this parser:
 .. _support-for-new-robots-parser:
 
 Implementing support for a new parser
--------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You can implement support for a new robots.txt_ parser by subclassing
 the abstract base class :class:`~scrapy.robotstxt.RobotParser` and

From 42056090516bb0cc5d349e232298c711ec452bc5 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Sun, 17 Jul 2022 15:50:40 +0500
Subject: [PATCH 634/716] Fixed intersphinx references

---
 docs/conf.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 9a0afe73e..3241295af 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -291,9 +291,10 @@ intersphinx_mapping = {
     'pytest': ('https://docs.pytest.org/en/latest', None),
     'python': ('https://docs.python.org/3', None),
     'sphinx': ('https://www.sphinx-doc.org/en/master', None),
-    'tox': ('https://tox.readthedocs.io/en/latest', None),
-    'twisted': ('https://twistedmatrix.com/documents/current', None),
-    'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
+    'tox': ('https://tox.wiki/en/latest/', None),
+    'twisted': ('https://docs.twisted.org/en/stable/', None),
+    'twistedapi': ('https://docs.twisted.org/en/stable/api/', None),
+    'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
 }
 intersphinx_disabled_reftypes = []
 

From b21c16099ed0acb0589677afd98c0ae3b78cd17d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 22 Jul 2022 19:18:33 +0500
Subject: [PATCH 635/716] Fix flake8 issues.

---
 tests/test_spidermiddleware.py | 6 +++---
 tests/test_utils_python.py     | 2 --
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index ed0912b82..edde6f682 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -305,8 +305,8 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
         with self.assertRaisesRegex(
             _InvalidOutput,
             (
-                "\.process_spider_output must return an iterable, got <class "
-                "'NoneType'>"
+                r"\.process_spider_output must return an iterable, got <class "
+                r"'NoneType'>"
             ),
         ):
             yield self._get_middleware_result(
@@ -317,7 +317,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
     def test_coroutine(self):
         with self.assertRaisesRegex(
             _InvalidOutput,
-            "\.process_spider_output must be an asynchronous generator",
+            r"\.process_spider_output must be an asynchronous generator",
         ):
             yield self._get_middleware_result(
                 ProcessSpiderOutputCoroutineMiddleware,
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 00b06b839..b1a8fdc04 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -2,8 +2,6 @@ import functools
 import gc
 import operator
 import platform
-import unittest
-from datetime import datetime
 from itertools import count
 from warnings import catch_warnings, filterwarnings
 

From af7dd16d8ded3e6cb2946603688f4f4a5212e80f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 25 Jul 2022 13:15:17 +0200
Subject: [PATCH 636/716] Merge pull request from GHSA-9x8m-2xpf-crp3

* Enforce matching proxy request meta and Proxy-Authorization header

* Cover proxy credential security fix in the release notes

* Remove extra empty line

* Reword the security issue description

* Address scenario where Proxy-Authorization is unexpectedly removed by a prior middleware

* Set the release date of Scrapy 2.6.2 and 1.8.3
---
 docs/news.rst                                | 106 ++++++-
 scrapy/downloadermiddlewares/httpproxy.py    |  56 ++--
 tests/test_downloadermiddleware_httpproxy.py | 313 ++++++++++++++++++-
 3 files changed, 439 insertions(+), 36 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 7993b4b4f..5bd9ca059 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5,10 +5,57 @@ Release notes
 
 .. _release-2.6.2:
 
-Scrapy 2.6.2 (to be determined)
--------------------------------
+Scrapy 2.6.2 (2022-07-25)
+-------------------------
 
-Fixes additional regressions introduced in 2.6.0:
+**Security bug fix:**
+
+-   When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
+    processes a request with :reqmeta:`proxy` metadata, and that
+    :reqmeta:`proxy` metadata includes proxy credentials,
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
+    the ``Proxy-Authentication`` header, but only if that header is not already
+    set.
+
+    There are third-party proxy-rotation downloader middlewares that set
+    different :reqmeta:`proxy` metadata every time they process a request.
+
+    Because of request retries and redirects, the same request can be processed
+    by downloader middlewares more than once, including both
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and
+    any third-party proxy-rotation downloader middleware.
+
+    These third-party proxy-rotation downloader middlewares could change the
+    :reqmeta:`proxy` metadata of a request to a new value, but fail to remove
+    the ``Proxy-Authentication`` header from the previous value of the
+    :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
+    to a different proxy.
+
+    To prevent the unintended leaking of proxy credentials, the behavior of
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now
+    as follows when processing a request:
+
+    -   If the request being processed defines :reqmeta:`proxy` metadata that
+        includes credentials, the ``Proxy-Authorization`` header is always
+        updated to feature those credentials.
+
+    -   If the request being processed defines :reqmeta:`proxy` metadata
+        without credentials, the ``Proxy-Authorization`` header is removed
+        *unless* it was originally defined for the same proxy URL.
+
+        To remove proxy credentials while keeping the same proxy URL, remove
+        the ``Proxy-Authorization`` header.
+
+    -   If the request has no :reqmeta:`proxy` metadata, or that metadata is a
+        falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is
+        removed.
+
+        It is no longer possible to set a proxy URL through the
+        :reqmeta:`proxy` metadata but set the credentials through the
+        ``Proxy-Authorization`` header. Set proxy credentials through the
+        :reqmeta:`proxy` metadata instead.
+
+Also fixes the following regressions introduced in 2.6.0:
 
 -   :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple
     spiders (:issue:`5435`, :issue:`5436`)
@@ -1925,6 +1972,59 @@ affect subclasses:
 (:issue:`3884`)
 
 
+.. _release-1.8.3:
+
+Scrapy 1.8.3 (2022-07-25)
+-------------------------
+
+**Security bug fix:**
+
+-   When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
+    processes a request with :reqmeta:`proxy` metadata, and that
+    :reqmeta:`proxy` metadata includes proxy credentials,
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
+    the ``Proxy-Authentication`` header, but only if that header is not already
+    set.
+
+    There are third-party proxy-rotation downloader middlewares that set
+    different :reqmeta:`proxy` metadata every time they process a request.
+
+    Because of request retries and redirects, the same request can be processed
+    by downloader middlewares more than once, including both
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and
+    any third-party proxy-rotation downloader middleware.
+
+    These third-party proxy-rotation downloader middlewares could change the
+    :reqmeta:`proxy` metadata of a request to a new value, but fail to remove
+    the ``Proxy-Authentication`` header from the previous value of the
+    :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
+    to a different proxy.
+
+    To prevent the unintended leaking of proxy credentials, the behavior of
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now
+    as follows when processing a request:
+
+    -   If the request being processed defines :reqmeta:`proxy` metadata that
+        includes credentials, the ``Proxy-Authorization`` header is always
+        updated to feature those credentials.
+
+    -   If the request being processed defines :reqmeta:`proxy` metadata
+        without credentials, the ``Proxy-Authorization`` header is removed
+        *unless* it was originally defined for the same proxy URL.
+
+        To remove proxy credentials while keeping the same proxy URL, remove
+        the ``Proxy-Authorization`` header.
+
+    -   If the request has no :reqmeta:`proxy` metadata, or that metadata is a
+        falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is
+        removed.
+
+        It is no longer possible to set a proxy URL through the
+        :reqmeta:`proxy` metadata but set the credentials through the
+        ``Proxy-Authorization`` header. Set proxy credentials through the
+        :reqmeta:`proxy` metadata instead.
+
+
 .. _release-1.8.2:
 
 Scrapy 1.8.2 (2022-03-01)
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index d2665b655..1deda42bd 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -45,31 +45,37 @@ class HttpProxyMiddleware:
         return creds, proxy_url
 
     def process_request(self, request, spider):
-        # ignore if proxy is already set
+        creds, proxy_url = None, None
         if 'proxy' in request.meta:
-            if request.meta['proxy'] is None:
-                return
-            # extract credentials if present
-            creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
+            if request.meta['proxy'] is not None:
+                creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
+        elif self.proxies:
+            parsed = urlparse_cached(request)
+            scheme = parsed.scheme
+            if (
+                (
+                    # 'no_proxy' is only supported by http schemes
+                    scheme not in ('http', 'https')
+                    or not proxy_bypass(parsed.hostname)
+                )
+                and scheme in self.proxies
+            ):
+                creds, proxy_url = self.proxies[scheme]
+
+        self._set_proxy_and_creds(request, proxy_url, creds)
+
+    def _set_proxy_and_creds(self, request, proxy_url, creds):
+        if proxy_url:
             request.meta['proxy'] = proxy_url
-            if creds and not request.headers.get('Proxy-Authorization'):
-                request.headers['Proxy-Authorization'] = b'Basic ' + creds
-            return
-        elif not self.proxies:
-            return
-
-        parsed = urlparse_cached(request)
-        scheme = parsed.scheme
-
-        # 'no_proxy' is only supported by http schemes
-        if scheme in ('http', 'https') and proxy_bypass(parsed.hostname):
-            return
-
-        if scheme in self.proxies:
-            self._set_proxy(request, scheme)
-
-    def _set_proxy(self, request, scheme):
-        creds, proxy = self.proxies[scheme]
-        request.meta['proxy'] = proxy
+        elif request.meta.get('proxy') is not None:
+            request.meta['proxy'] = None
         if creds:
-            request.headers['Proxy-Authorization'] = b'Basic ' + creds
+            request.headers[b'Proxy-Authorization'] = b'Basic ' + creds
+            request.meta['_auth_proxy'] = proxy_url
+        elif '_auth_proxy' in request.meta:
+            if proxy_url != request.meta['_auth_proxy']:
+                if b'Proxy-Authorization' in request.headers:
+                    del request.headers[b'Proxy-Authorization']
+                del request.meta['_auth_proxy']
+        elif b'Proxy-Authorization' in request.headers:
+            del request.headers[b'Proxy-Authorization']
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 7c97bf32a..67134cf93 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -65,12 +65,12 @@ class TestHttpProxyMiddleware(TestCase):
         mw = HttpProxyMiddleware()
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz')
         # proxy from request.meta
         req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=')
 
     def test_proxy_auth_empty_passwd(self):
@@ -78,12 +78,12 @@ class TestHttpProxyMiddleware(TestCase):
         mw = HttpProxyMiddleware()
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')
         # proxy from request.meta
         req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6')
 
     def test_proxy_auth_encoding(self):
@@ -92,26 +92,26 @@ class TestHttpProxyMiddleware(TestCase):
         mw = HttpProxyMiddleware(auth_encoding='utf-8')
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
 
         # proxy from request.meta
         req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
 
         # default latin-1 encoding
         mw = HttpProxyMiddleware(auth_encoding='latin-1')
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
 
         # proxy from request.meta, latin-1 encoding
         req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
 
     def test_proxy_already_seted(self):
@@ -152,3 +152,300 @@ class TestHttpProxyMiddleware(TestCase):
         # '/var/run/docker.sock' may be used by the user for
         # no_proxy value but is not parseable and should be skipped
         assert 'no' not in mw.proxies
+
+    def test_add_proxy_without_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request('https://example.com')
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = 'https://example.com'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_add_proxy_with_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request('https://example.com')
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = 'https://user1:password1@example.com'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        encoded_credentials = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+    def test_remove_proxy_without_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = None
+        assert middleware.process_request(request, spider) is None
+        self.assertIsNone(request.meta['proxy'])
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_remove_proxy_with_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = None
+        assert middleware.process_request(request, spider) is None
+        self.assertIsNone(request.meta['proxy'])
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_add_credentials(self):
+        """If the proxy request meta switches to a proxy URL with the same
+        proxy and adds credentials (there were no credentials before), the new
+        credentials must be used."""
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+
+        request.meta['proxy'] = 'https://user1:password1@example.com'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        encoded_credentials = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+    def test_change_credentials(self):
+        """If the proxy request meta switches to a proxy URL with different
+        credentials, those new credentials must be used."""
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = 'https://user2:password2@example.com'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        encoded_credentials = middleware._basic_auth_header(
+            'user2',
+            'password2',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+    def test_remove_credentials(self):
+        """If the proxy request meta switches to a proxy URL with the same
+        proxy but no credentials, the original credentials must be still
+        used.
+
+        To remove credentials while keeping the same proxy URL, users must
+        delete the Proxy-Authorization header.
+        """
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+
+        request.meta['proxy'] = 'https://example.com'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        encoded_credentials = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+        request.meta['proxy'] = 'https://example.com'
+        del request.headers[b'Proxy-Authorization']
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_change_proxy_add_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+
+        request.meta['proxy'] = 'https://user1:password1@example.org'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.org')
+        encoded_credentials = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+    def test_change_proxy_keep_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+
+        request.meta['proxy'] = 'https://user1:password1@example.org'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.org')
+        encoded_credentials = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+        # Make sure, indirectly, that _auth_proxy is updated.
+        request.meta['proxy'] = 'https://example.com'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_change_proxy_change_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+
+        request.meta['proxy'] = 'https://user2:password2@example.org'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.org')
+        encoded_credentials = middleware._basic_auth_header(
+            'user2',
+            'password2',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+    def test_change_proxy_remove_credentials(self):
+        """If the proxy request meta switches to a proxy URL with a different
+        proxy and no credentials, no credentials must be used."""
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = 'https://example.org'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta, {'proxy': 'https://example.org'})
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_change_proxy_remove_credentials_preremoved_header(self):
+        """Corner case of proxy switch with credentials removal where the
+        credentials have been removed beforehand.
+
+        It ensures that our implementation does not assume that the credentials
+        header exists when trying to remove it.
+        """
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = 'https://example.org'
+        del request.headers[b'Proxy-Authorization']
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta, {'proxy': 'https://example.org'})
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_proxy_authentication_header_undefined_proxy(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            headers={'Proxy-Authorization': 'Basic foo'},
+        )
+        assert middleware.process_request(request, spider) is None
+        self.assertNotIn('proxy', request.meta)
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_proxy_authentication_header_disabled_proxy(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            headers={'Proxy-Authorization': 'Basic foo'},
+            meta={'proxy': None},
+        )
+        assert middleware.process_request(request, spider) is None
+        self.assertIsNone(request.meta['proxy'])
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_proxy_authentication_header_proxy_without_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            headers={'Proxy-Authorization': 'Basic foo'},
+            meta={'proxy': 'https://example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_proxy_authentication_header_proxy_with_same_credentials(self):
+        middleware = HttpProxyMiddleware()
+        encoded_credentials = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        request = Request(
+            'https://example.com',
+            headers={'Proxy-Authorization': b'Basic ' + encoded_credentials},
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+    def test_proxy_authentication_header_proxy_with_different_credentials(self):
+        middleware = HttpProxyMiddleware()
+        encoded_credentials1 = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        request = Request(
+            'https://example.com',
+            headers={'Proxy-Authorization': b'Basic ' + encoded_credentials1},
+            meta={'proxy': 'https://user2:password2@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        encoded_credentials2 = middleware._basic_auth_header(
+            'user2',
+            'password2',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials2,
+        )

From aecbccbaa567b07694141a4503e9abf1bb2c919f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 25 Jul 2022 13:33:23 +0200
Subject: [PATCH 637/716] =?UTF-8?q?Bump=20version:=202.6.1=20=E2=86=92=202?=
 =?UTF-8?q?.6.2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 scrapy/VERSION   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 1d9b9c02f..2e2f7949a 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.6.1
+current_version = 2.6.2
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 6a6a3d8e3..097a15a2a 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.6.1
+2.6.2

From 5862f6b8e1cd9e240c4b3e79f2eddd0b9f33ef8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 25 Jul 2022 17:25:10 +0200
Subject: [PATCH 638/716] Update frozen CI packages (#5574)

---
 pylintrc |  6 +-----
 tox.ini  | 13 ++++++++-----
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/pylintrc b/pylintrc
index 2cdd6321e..d8e47dc11 100644
--- a/pylintrc
+++ b/pylintrc
@@ -9,11 +9,9 @@ disable=abstract-method,
         arguments-renamed,
         attribute-defined-outside-init,
         bad-classmethod-argument,
-        bad-continuation,
         bad-indentation,
         bad-mcs-classmethod-argument,
         bad-super-call,
-        bad-whitespace,
         bare-except,
         blacklisted-name,
         broad-except,
@@ -52,7 +50,6 @@ disable=abstract-method,
         logging-not-lazy,
         lost-exception,
         method-hidden,
-        misplaced-comparison-constant,
         missing-docstring,
         missing-final-newline,
         multiple-imports,
@@ -60,12 +57,10 @@ disable=abstract-method,
         no-else-continue,
         no-else-raise,
         no-else-return,
-        no-init,
         no-member,
         no-method-argument,
         no-name-in-module,
         no-self-argument,
-        no-self-use,
         no-value-for-parameter,
         not-an-iterable,
         not-callable,
@@ -102,6 +97,7 @@ disable=abstract-method,
         ungrouped-imports,
         unidiomatic-typecheck,
         unnecessary-comprehension,
+        unnecessary-dunder-call,
         unnecessary-lambda,
         unnecessary-pass,
         unreachable,
diff --git a/tox.ini b/tox.ini
index ab8a715c2..4d1bb574d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -47,7 +47,7 @@ commands =
 [testenv:security]
 basepython = python3
 deps =
-    bandit==1.7.3
+    bandit==1.7.4
 commands =
     bandit -r -c .bandit.yml {posargs:scrapy}
 
@@ -57,16 +57,17 @@ deps =
     {[testenv]deps}
     # Twisted[http2] is required to import some files
     Twisted[http2]>=17.9.0
-    pytest-flake8
-    flake8==3.9.2  # https://github.com/tholo/pytest-flake8/issues/81
+    pytest-flake8==1.1.1
+    flake8==4.0.1
 commands =
     pytest --flake8 {posargs:docs scrapy tests}
 
 [testenv:pylint]
-basepython = python3
+# reppy does not support Python 3.9+
+basepython = python3.8
 deps =
     {[testenv:extra-deps]deps}
-    pylint==2.12.2
+    pylint==2.14.5
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 
@@ -117,6 +118,8 @@ setenv =
     {[pinned]setenv}
 
 [testenv:extra-deps]
+# reppy does not support Python 3.9+
+basepython = python3.8
 deps =
     {[testenv]deps}
     boto

From 56e2eeac1066cd2d3c710b76a3d6210b3ac257f5 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 27 Jul 2022 09:41:12 +0500
Subject: [PATCH 639/716] fix typing issues by upgrading mypy

---
 tests/test_utils_request.py | 2 +-
 tox.ini                     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 5ee772c0b..8bc7922b6 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -52,7 +52,7 @@ class UtilsRequestTest(unittest.TestCase):
 class FingerprintTest(unittest.TestCase):
     maxDiff = None
 
-    function = staticmethod(fingerprint)
+    function: staticmethod = staticmethod(fingerprint)
     cache: Union[
         "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]",
         "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]",
diff --git a/tox.ini b/tox.ini
index 4d1bb574d..2110e1020 100644
--- a/tox.ini
+++ b/tox.ini
@@ -38,7 +38,7 @@ install_command =
 basepython = python3
 deps =
     lxml-stubs==0.2.0
-    mypy==0.910
+    mypy==0.971
     types-pyOpenSSL==20.0.3
     types-setuptools==57.0.0
 commands =

From 67011cd9576b02d696c7e0c0dc7f76074e30901b Mon Sep 17 00:00:00 2001
From: Aftab Alam <88653530+itsAftabAlam@users.noreply.github.com>
Date: Wed, 27 Jul 2022 10:20:26 +0530
Subject: [PATCH 640/716] updated README.rst , added hyperlink to banner
 (#5284)

Co-authored-by: Mikhail Korobov <kmike84@gmail.com>
---
 README.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.rst b/README.rst
index b543a30f4..358302c76 100644
--- a/README.rst
+++ b/README.rst
@@ -1,4 +1,5 @@
 .. image:: https://scrapy.org/img/scrapylogo.png
+   :target: https://scrapy.org/
    
 ======
 Scrapy

From 83ecdf1bcab310be5f2c59e5c005b8968d72a72c Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Wed, 27 Jul 2022 23:12:31 +0500
Subject: [PATCH 641/716] Update docs/topics/components.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/components.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/components.rst b/docs/topics/components.rst
index 1fff2d61a..c44f3def2 100644
--- a/docs/topics/components.rst
+++ b/docs/topics/components.rst
@@ -47,7 +47,7 @@ Enforcing component requirements
 ================================
 
 Sometimes, your components may only be intended to work under certain
-conditions. For example, the may require a minimum version of Scrapy to work as
+conditions. For example, they may require a minimum version of Scrapy to work as
 intended, or they may require certain settings to have specific values.
 
 In addition to describing those conditions in the documentation of your

From 0f1112f3e22d91e505f736fc78fae94eda07c72d Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Wed, 27 Jul 2022 23:12:43 +0500
Subject: [PATCH 642/716] Update docs/index.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/index.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/index.rst b/docs/index.rst
index ea4950e4c..5404969e0 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -258,7 +258,7 @@ Extending Scrapy
     components.
 
 :doc:`topics/api`
-    Use it on extensions and middlewares to extend Scrapy functionality
+    Use it on extensions and middlewares to extend Scrapy functionality.
 
 
 All the rest

From c7b90c6e1e3de20256d8bf6fc5d18da44d562b0d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 28 Jul 2022 13:44:36 +0500
Subject: [PATCH 643/716] Extract more common code.

---
 scrapy/spidermiddlewares/depth.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 29634c3ad..4c923b1b3 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -28,25 +28,22 @@ class DepthMiddleware:
         return cls(maxdepth, crawler.stats, verbose, prio)
 
     def process_spider_output(self, response, result, spider):
-        # base case (depth=0)
-        if 'depth' not in response.meta:
-            response.meta['depth'] = 0
-            if self.verbose_stats:
-                self.stats.inc_value('request_depth_count/0', spider=spider)
-
+        self._init_depth(response, spider)
         return (r for r in result or () if self._filter(r, response, spider))
 
     async def process_spider_output_async(self, response, result, spider):
+        self._init_depth(response, spider)
+        async for r in result or ():
+            if self._filter(r, response, spider):
+                yield r
+
+    def _init_depth(self, response, spider):
         # base case (depth=0)
         if 'depth' not in response.meta:
             response.meta['depth'] = 0
             if self.verbose_stats:
                 self.stats.inc_value('request_depth_count/0', spider=spider)
 
-        async for r in result or ():
-            if self._filter(r, response, spider):
-                yield r
-
     def _filter(self, request, response, spider):
         if not isinstance(request, Request):
             return True

From 13c5ad7e688271ae47f2a489bf665cb15387b3a3 Mon Sep 17 00:00:00 2001
From: "Alexandr N. Zamaraev" <tonal.promsoft@gmail.com>
Date: Tue, 16 Aug 2022 11:03:37 +0700
Subject: [PATCH 644/716] Partial in is_generator_with_return_value

See path in #5592
---
 scrapy/utils/misc.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 1221b39b2..2e25f6421 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -9,6 +9,7 @@ from collections import deque
 from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
+from functools import partial
 
 from w3lib.html import replace_entities
 
@@ -226,7 +227,11 @@ def is_generator_with_return_value(callable):
         return value is None or isinstance(value, ast.NameConstant) and value.value is None
 
     if inspect.isgeneratorfunction(callable):
-        code = re.sub(r"^[\t ]+", "", inspect.getsource(callable))
+        func = callable
+        while isinstance(func, partial):
+            func = func.func
+
+        code = re.sub(r"^[\t ]+", "", inspect.getsource(func))
         tree = ast.parse(code)
         for node in walk_callable(tree):
             if isinstance(node, ast.Return) and not returns_none(node):

From c49b5aaf77a668e44bffb54c52310084e4068162 Mon Sep 17 00:00:00 2001
From: zaid-ismail031 <ab1cd2eefre3@gmail.com>
Date: Thu, 25 Aug 2022 02:17:03 +0200
Subject: [PATCH 645/716] Changed incorrect information regarding the return
 type of parse/request callback method.

---
 docs/topics/spiders.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index ece02ae47..ffe41cf3e 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -181,9 +181,10 @@ scrapy.Spider
        scraped data and/or more URLs to follow. Other Requests callbacks have
        the same requirements as the :class:`Spider` class.
 
-       This method, as well as any other Request callback, must return an
-       iterable of :class:`~scrapy.Request` and/or :ref:`item objects
-       <topics-items>`.
+       This method, as well as any other Request callback, must return a
+       :class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, an
+       iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects
+       <topics-items>`, or ``None``.
 
        :param response: the response to parse
        :type response: :class:`~scrapy.http.Response`

From e411ea94eb853eedeee111469c6442d275864a09 Mon Sep 17 00:00:00 2001
From: Mohammadtaher Abbasi <mohammadtaher.abbasi@zoodfood.com>
Date: Sun, 28 Aug 2022 20:28:13 +0430
Subject: [PATCH 646/716] BOM should take precedence over Content-Type header
 when detecting the encoding closes #5601

---
 scrapy/http/response/text.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 89516b9b6..bfcde878d 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -11,8 +11,13 @@ from typing import Generator, Tuple
 from urllib.parse import urljoin
 
 import parsel
-from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
-                            http_content_type_encoding, resolve_encoding)
+from w3lib.encoding import (
+    html_body_declared_encoding,
+    html_to_unicode,
+    http_content_type_encoding,
+    resolve_encoding,
+    read_bom,
+)
 from w3lib.html import strip_html5_whitespace
 
 from scrapy.http import Request
@@ -60,6 +65,7 @@ class TextResponse(Response):
     def _declared_encoding(self):
         return (
             self._encoding
+            or self._bom_encoding()
             or self._headers_encoding()
             or self._body_declared_encoding()
         )
@@ -117,6 +123,10 @@ class TextResponse(Response):
     def _body_declared_encoding(self):
         return html_body_declared_encoding(self.body)
 
+    @memoizemethod_noargs
+    def _bom_encoding(self):
+        return read_bom(self.body)[0]
+
     @property
     def selector(self):
         from scrapy.selector import Selector

From a988c4b78b9ca104b74cff60c46c1842e3f25652 Mon Sep 17 00:00:00 2001
From: Mohammadtaher Abbasi <mohammadtaher.abbasi@zoodfood.com>
Date: Mon, 29 Aug 2022 17:08:30 +0430
Subject: [PATCH 647/716] add test

---
 tests/test_http_response.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 2986f884f..5d67a5e74 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,3 +1,4 @@
+import codecs
 import unittest
 from unittest import mock
 
@@ -358,6 +359,8 @@ class TextResponseTest(BaseResponseTest):
                                  headers={"Content-type": ["text/html; charset=gb2312"]})
         r7 = self.response_class("http://www.example.com", body=b"\xa8D",
                                  headers={"Content-type": ["text/html; charset=gbk"]})
+        r8 = self.response_class("http://www.example.com", body=codecs.BOM_UTF8 + b"\xc2\xa3",
+                                 headers={"Content-type": ["text/html; charset=cp1251"]})
 
         self.assertEqual(r1._headers_encoding(), "utf-8")
         self.assertEqual(r2._headers_encoding(), None)
@@ -367,7 +370,10 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r3._declared_encoding(), "cp1252")
         self.assertEqual(r4._headers_encoding(), None)
         self.assertEqual(r5._headers_encoding(), None)
+        self.assertEqual(r8._headers_encoding(), "cp1251")
+        self.assertEqual(r8._declared_encoding(), "utf-8")
         self._assert_response_encoding(r5, "utf-8")
+        self._assert_response_encoding(r8, "utf-8")
         assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'
         self._assert_response_values(r1, 'utf-8', "\xa3")
         self._assert_response_values(r2, 'utf-8', "\xa3")

From f4bcc3e67de0896999be48d1b0fcede0be0d69e4 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 6 Sep 2022 01:15:41 -0300
Subject: [PATCH 648/716] fix: failed tests

---
 tests/test_crawler.py | 13 +++++++++----
 tests/test_spider.py  | 12 ++++++++----
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 4e9ef7740..4e599d69c 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -23,6 +23,8 @@ from scrapy.utils.test import get_crawler
 from scrapy.extensions.throttle import AutoThrottle
 from scrapy.extensions import telnet
 from scrapy.utils.test import get_testenv
+from pkg_resources import parse_version
+from w3lib import __version__ as w3lib_version
 
 from tests.mockserver import MockServer
 
@@ -381,10 +383,13 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
     def test_ipv6_default_name_resolver(self):
         log = self.run_script('default_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
-        self.assertIn(
-            "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
-            log)
+        if parse_version(w3lib_version) < parse_version("2.0.0"):
+            self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
+            self.assertIn(
+                "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
+                log)
+        else:
+            self.assertIn("ValueError: invalid hostname:", log)
 
     def test_caching_hostname_resolver_ipv6(self):
         log = self.run_script("caching_hostname_resolver_ipv6.py")
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 689349999..7b36304b1 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -22,6 +22,8 @@ from scrapy.spiders import (
 from scrapy.linkextractors import LinkExtractor
 from scrapy.utils.test import get_crawler
 from tests import get_testdata
+from pkg_resources import parse_version
+from w3lib import __version__ as w3lib_version
 
 
 class SpiderTest(unittest.TestCase):
@@ -360,10 +362,12 @@ class CrawlSpiderTest(SpiderTest):
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
-                          'http://EXAMPLE.ORG/ABOUT.HTML',
-                          'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
+        urls = ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
+                'http://EXAMPLE.ORG/ABOUT.HTML',
+                'http://EXAMPLE.ORG/NOFOLLOW.HTML']
+        if parse_version(w3lib_version) >= parse_version('2.0.0'):
+            urls = list(map(lambda u: u.replace("EXAMPLE.ORG", "example.org"), urls))
+        self.assertEqual([r.url for r in output], urls)
 
     def test_process_request_instance_method_with_response(self):
 

From 1289422284991c09cbee8cea17bef837b3efc31b Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 6 Sep 2022 08:17:58 -0300
Subject: [PATCH 649/716] chore: Skip `test_ipv6_default_name_resolver` test if
 w3lib version >= 2.0.0

---
 tests/test_crawler.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 4e599d69c..e2a14be55 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -380,16 +380,15 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn('Spider closed (finished)', log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    @mark.skipif(parse_version(w3lib_version) >= parse_version("2.0.0"),
+                 reason='w3lib 2.0.0 and later do not allow invalid domains.')
     def test_ipv6_default_name_resolver(self):
         log = self.run_script('default_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
-        if parse_version(w3lib_version) < parse_version("2.0.0"):
-            self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
-            self.assertIn(
-                "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
-                log)
-        else:
-            self.assertIn("ValueError: invalid hostname:", log)
+        self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
+        self.assertIn(
+            "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
+            log)
 
     def test_caching_hostname_resolver_ipv6(self):
         log = self.run_script("caching_hostname_resolver_ipv6.py")

From 582a6bf6dbcb01da40cbec6b51269add2db39cf1 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 6 Sep 2022 10:03:18 -0300
Subject: [PATCH 650/716] refactor: Use `safe_url_string` to standardize url
 output

---
 tests/test_spider.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index 7b36304b1..e1527620f 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -22,8 +22,7 @@ from scrapy.spiders import (
 from scrapy.linkextractors import LinkExtractor
 from scrapy.utils.test import get_crawler
 from tests import get_testdata
-from pkg_resources import parse_version
-from w3lib import __version__ as w3lib_version
+from w3lib.url import safe_url_string
 
 
 class SpiderTest(unittest.TestCase):
@@ -362,12 +361,10 @@ class CrawlSpiderTest(SpiderTest):
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        urls = ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
-                'http://EXAMPLE.ORG/ABOUT.HTML',
-                'http://EXAMPLE.ORG/NOFOLLOW.HTML']
-        if parse_version(w3lib_version) >= parse_version('2.0.0'):
-            urls = list(map(lambda u: u.replace("EXAMPLE.ORG", "example.org"), urls))
-        self.assertEqual([r.url for r in output], urls)
+        self.assertEqual([r.url for r in output],
+                         [safe_url_string('http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML'),
+                          safe_url_string('http://EXAMPLE.ORG/ABOUT.HTML'),
+                          safe_url_string('http://EXAMPLE.ORG/NOFOLLOW.HTML')])
 
     def test_process_request_instance_method_with_response(self):
 

From ce0ca51485545aed8eb33a285bc3d2fbf8a8e407 Mon Sep 17 00:00:00 2001
From: "Magsen (CD)" <magsen01@hotmail.fr>
Date: Tue, 13 Sep 2022 12:07:58 +0200
Subject: [PATCH 651/716] fix: typo in tutorial

fix: typo in tutorial
---
 docs/intro/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 75928077e..092123d1d 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -379,7 +379,7 @@ like this:
 Let's open up scrapy shell and play a bit to find out how to extract the data
 we want::
 
-    $ scrapy shell 'https://quotes.toscrape.com'
+    scrapy shell 'https://quotes.toscrape.com'
 
 We get a list of selectors for the quote HTML elements with:
 

From 77c055ee28a767b2c8222274cc7556ab9cc56edc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 14 Sep 2022 14:47:14 +0200
Subject: [PATCH 652/716] Relax Proxy-Authorization restrictions

---
 scrapy/downloadermiddlewares/httpproxy.py    |  5 ++++-
 tests/test_downloadermiddleware_httpproxy.py | 14 +++++++++++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 1deda42bd..dd8a7e797 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -78,4 +78,7 @@ class HttpProxyMiddleware:
                     del request.headers[b'Proxy-Authorization']
                 del request.meta['_auth_proxy']
         elif b'Proxy-Authorization' in request.headers:
-            del request.headers[b'Proxy-Authorization']
+            if proxy_url:
+                request.meta['_auth_proxy'] = proxy_url
+            else:
+                del request.headers[b'Proxy-Authorization']
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 70eb94d77..44434f90e 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -400,6 +400,9 @@ class TestHttpProxyMiddleware(TestCase):
         self.assertNotIn(b'Proxy-Authorization', request.headers)
 
     def test_proxy_authentication_header_proxy_without_credentials(self):
+        """As long as the proxy URL in request metadata remains the same, the
+        Proxy-Authorization header is used and kept, and may even be
+        changed."""
         middleware = HttpProxyMiddleware()
         request = Request(
             'https://example.com',
@@ -408,7 +411,16 @@ class TestHttpProxyMiddleware(TestCase):
         )
         assert middleware.process_request(request, spider) is None
         self.assertEqual(request.meta['proxy'], 'https://example.com')
-        self.assertNotIn(b'Proxy-Authorization', request.headers)
+        self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo')
+
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo')
+
+        request.headers['Proxy-Authorization'] = b'Basic bar'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.headers['Proxy-Authorization'], b'Basic bar')
 
     def test_proxy_authentication_header_proxy_with_same_credentials(self):
         middleware = HttpProxyMiddleware()

From 1429aa011ce2a7a43691fc359c88bf40e512176c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= <gliptak@gmail.com>
Date: Tue, 20 Sep 2022 12:47:20 -0400
Subject: [PATCH 653/716] Update test-standard link in contributing docs
 (#5631)

---
 docs/contributing.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/contributing.rst b/docs/contributing.rst
index 946bdc23e..9cfe10012 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -214,7 +214,7 @@ Tests
 =====
 
 Tests are implemented using the :doc:`Twisted unit-testing framework
-<twisted:core/development/policy/test-standard>`. Running tests requires
+<twisted:development/test-standard>`. Running tests requires
 :doc:`tox <tox:index>`.
 
 .. _running-tests:

From 5f194202114fd38530c78299d51b6966b4802f59 Mon Sep 17 00:00:00 2001
From: Tim B <tim.benger@gmail.com>
Date: Wed, 21 Sep 2022 07:27:27 +0100
Subject: [PATCH 654/716] Documented how settings must be picklable (#5629)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
Co-authored-by: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
---
 docs/topics/settings.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 6722ce9ed..90f13f3ef 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -98,6 +98,10 @@ class.
 The global defaults are located in the ``scrapy.settings.default_settings``
 module and documented in the :ref:`topics-settings-ref` section.
 
+Compatibility with pickle
+=========================
+
+Setting values must be :ref:`picklable <pickle-picklable>`.
 
 Import paths and classes
 ========================

From 385acd5598fb06a126ac485087b58a4581096876 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= <gliptak@gmail.com>
Date: Sat, 24 Sep 2022 14:58:14 -0400
Subject: [PATCH 655/716] Match pyOpenSSL and service_identity to Twisted
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Gábor Lipták <gliptak@gmail.com>
---
 setup.py              | 6 +++---
 tests/test_crawler.py | 2 +-
 tox.ini               | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/setup.py b/setup.py
index ed197273f..fcc902c62 100644
--- a/setup.py
+++ b/setup.py
@@ -20,13 +20,13 @@ def has_environment_marker_platform_impl_support():
 
 install_requires = [
     'Twisted>=18.9.0',
-    'cryptography>=2.8',
+    'cryptography>=3.3',
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
     'parsel>=1.5.0',
-    'pyOpenSSL>=19.1.0',
+    'pyOpenSSL>=21.0.0',
     'queuelib>=1.4.2',
-    'service_identity>=16.0.0',
+    'service_identity>=18.1.0',
     'w3lib>=1.17.0',
     'zope.interface>=5.1.0',
     'protego>=0.1.15',
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index e2a14be55..cf15ba9b9 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -327,7 +327,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
 
     def test_reactor_default_twisted_reactor_select(self):
         log = self.run_script('reactor_default_twisted_reactor_select.py')
-        if platform.system() == 'Windows':
+        if platform.system() in ['Windows', 'Darwin']:
             # The goal of this test function is to test that, when a reactor is
             # installed (the default one here) and a different reactor is
             # configured (select here), an error raises.
diff --git a/tox.ini b/tox.ini
index 2110e1020..2bf9454d0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -73,15 +73,15 @@ commands =
 
 [pinned]
 deps =
-    cryptography==2.8
+    cryptography==3.3
     cssselect==0.9.1
     h2==3.0
     itemadapter==0.1.0
     parsel==1.5.0
     Protego==0.1.15
-    pyOpenSSL==19.1.0
+    pyOpenSSL==21.0.0
     queuelib==1.4.2
-    service_identity==16.0.0
+    service_identity==18.1.0
     Twisted[http2]==18.9.0
     w3lib==1.17.0
     zope.interface==5.1.0

From 79a4bc3da02bfe2b07c596f6b677760b2c04f96f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= <gliptak@gmail.com>
Date: Sun, 25 Sep 2022 14:17:57 -0400
Subject: [PATCH 656/716] Cleanup METHOD_SSLv3
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Gábor Lipták <gliptak@gmail.com>
---
 docs/topics/settings.rst                 | 1 -
 scrapy/core/downloader/contextfactory.py | 2 +-
 scrapy/core/downloader/tls.py            | 2 --
 3 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 90f13f3ef..a711fd197 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -564,7 +564,6 @@ This setting must be one of these string values:
   set this if you want the behavior of Scrapy<1.1
 - ``'TLSv1.1'``: forces TLS version 1.1
 - ``'TLSv1.2'``: forces TLS version 1.2
-- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
 
 
 .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index b5318c7bb..4abde2238 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -21,7 +21,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
     which allows TLS protocol negotiation
 
     'A TLS/SSL connection established with [this method] may
-     understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
+     understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.'
     """
 
     def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 19a56d9b6..698a1c85c 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -11,7 +11,6 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
 logger = logging.getLogger(__name__)
 
 
-METHOD_SSLv3 = 'SSLv3'
 METHOD_TLS = 'TLS'
 METHOD_TLSv10 = 'TLSv1.0'
 METHOD_TLSv11 = 'TLSv1.1'
@@ -20,7 +19,6 @@ METHOD_TLSv12 = 'TLSv1.2'
 
 openssl_methods = {
     METHOD_TLS: SSL.SSLv23_METHOD,                      # protocol negotiation (recommended)
-    METHOD_SSLv3: SSL.SSLv3_METHOD,                     # SSL 3 (NOT recommended)
     METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only
     METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
     METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only

From 1d79994dccf396caca93f53cd50646281fee1def Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 27 Sep 2022 17:01:33 +0200
Subject: [PATCH 657/716] Copy 2.6.3 release notes from the 2.6 branch

---
 docs/news.rst | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index d27c105a5..9469d0fe5 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,32 @@
 Release notes
 =============
 
+.. _release-2.6.3:
+
+Scrapy 2.6.3 (2022-09-27)
+-------------------------
+
+-   Added support for pyOpenSSL_ 22.1.0, removing support for SSLv3
+    (:issue:`5634`, :issue:`5635`, :issue:`5636`).
+
+-   Upgraded the minimum versions of the following dependencies:
+
+    -   cryptography_: 2.0 → 3.3
+
+    -   pyOpenSSL_: 16.2.0 → 21.0.0
+
+    -   service_identity_: 16.0.0 → 18.1.0
+
+    -   Twisted_: 17.9.0 → 18.9.0
+
+    -   zope.interface_: 4.1.3 → 5.0.0
+
+    (:issue:`5621`, :issue:`5632`)
+
+-   Fixes test and documentation issues (:issue:`5612`, :issue:`5617`,
+    :issue:`5631`).
+
+
 .. _release-2.6.2:
 
 Scrapy 2.6.2 (2022-07-25)

From 3ca7877781fdae5415c57a9a0f348b1797795209 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Thu, 29 Sep 2022 11:51:11 -0300
Subject: [PATCH 658/716] chore: Skip `batch_path_differ` test on Windows

---
 tests/test_feedexport.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index ecd1b59d3..ad2383018 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2285,6 +2285,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             for expected_batch, got_batch in zip(expected, data[fmt]):
                 self.assertEqual(expected_batch, got_batch)
 
+    @pytest.mark.skipif(sys.platform == 'win32', reason='Odd behaviour on file creation/output')
     @defer.inlineCallbacks
     def test_batch_path_differ(self):
         """
@@ -2305,7 +2306,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
-        self.assertEqual(len(items) + 1, len(data['json']))
+        self.assertEqual(len(items), len([_ for _ in data['json'] if _]))
 
     @defer.inlineCallbacks
     def test_stats_batch_file_success(self):

From 9f006e3aa595acc6499f08a90696d7cd0dc0bca7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= <gliptak@gmail.com>
Date: Sun, 25 Sep 2022 13:48:51 -0400
Subject: [PATCH 659/716] Match pyOpenSSL and types-pyOpenSSL versions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Gábor Lipták <gliptak@gmail.com>
---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 2bf9454d0..0a47a1c0b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -39,7 +39,7 @@ basepython = python3
 deps =
     lxml-stubs==0.2.0
     mypy==0.971
-    types-pyOpenSSL==20.0.3
+    types-pyOpenSSL==21.0.0
     types-setuptools==57.0.0
 commands =
     mypy --show-error-codes {posargs: scrapy tests}

From 116d9a97481f5ee4028a1f3f8e72ca34b35a0be9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= <gliptak@gmail.com>
Date: Fri, 30 Sep 2022 22:58:14 -0400
Subject: [PATCH 660/716] Correct distutils deprecation warning
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Gábor Lipták <gliptak@gmail.com>
---
 scrapy/utils/display.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index f4d17224b..d28df40c7 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -5,7 +5,7 @@ pprint and pformat wrappers with colorization support
 import ctypes
 import platform
 import sys
-from distutils.version import LooseVersion as parse_version
+from packaging.version import Version as parse_version
 from pprint import pformat as pformat_
 
 

From c3f35d2ad79d19e7e000b8ba0df7d1d9f10658b2 Mon Sep 17 00:00:00 2001
From: Oscar Dominguez <dominguez.celada@gmail.com>
Date: Sat, 1 Oct 2022 19:37:51 +0200
Subject: [PATCH 661/716] ci(tests-windows): upgrade actions/setup-python to v4

---
 .github/workflows/tests-windows.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 955b9b449..0d85a8e09 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -28,7 +28,7 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 

From 9fcbf3bcbc8b8d09bb7d8b246dae9156d6848a5b Mon Sep 17 00:00:00 2001
From: Oscar Dominguez <dominguez.celada@gmail.com>
Date: Sat, 1 Oct 2022 19:38:16 +0200
Subject: [PATCH 662/716] ci(tests-windows): upgrade actions/checkout to v3

---
 .github/workflows/tests-windows.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 955b9b449..2d8c140af 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -25,7 +25,7 @@ jobs:
             TOXENV: asyncio
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2

From 1a2cb61e22506c47e78bc800739d9cb2a8bf0144 Mon Sep 17 00:00:00 2001
From: Oscar Dominguez <dominguez.celada@gmail.com>
Date: Sat, 1 Oct 2022 20:27:10 +0200
Subject: [PATCH 663/716] ci(test-macos): upgrade actions/checkout to v3

---
 .github/workflows/tests-macos.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 7819a4e12..8c9ef5c2f 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -10,7 +10,7 @@ jobs:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2

From af95331296a91444947095af19f8244d8bc915e1 Mon Sep 17 00:00:00 2001
From: Oscar Dominguez <dominguez.celada@gmail.com>
Date: Sat, 1 Oct 2022 20:27:23 +0200
Subject: [PATCH 664/716] ci(test-macos): upgrade actions/setup-python to v4

---
 .github/workflows/tests-macos.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 7819a4e12..d49272fbf 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -13,7 +13,7 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 

From 759ad5dee4120dc2197ea6fa1559c92f77472abf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= <gliptak@gmail.com>
Date: Sun, 2 Oct 2022 09:09:04 -0400
Subject: [PATCH 665/716] Require packaging

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index fcc902c62..8b9c43738 100644
--- a/setup.py
+++ b/setup.py
@@ -32,6 +32,7 @@ install_requires = [
     'protego>=0.1.15',
     'itemadapter>=0.1.0',
     'setuptools',
+    'packaging',
     'tldextract',
     'lxml>=4.3.0',
 ]

From c7d800ab229df50c2767ad67460e90f6213fc293 Mon Sep 17 00:00:00 2001
From: Abdul Rauf <abdulraufmujahid@gmail.com>
Date: Sun, 2 Oct 2022 19:12:48 +0500
Subject: [PATCH 666/716] CI: add Twine check in check workflow

---
 .github/workflows/checks.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index b26f344ff..a708474ef 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -39,3 +39,8 @@ jobs:
       run: |
         pip install -U tox
         tox
+    - name: Twine check
+      run: |
+        pip install twine
+        python setup.py sdist
+        twine check dist/*

From 69bf5c662555db64979a90c5c284b1faa4f24992 Mon Sep 17 00:00:00 2001
From: Abdul Rauf <abdulraufmujahid@gmail.com>
Date: Sun, 2 Oct 2022 20:27:24 +0500
Subject: [PATCH 667/716] CI: move twinecheck to tox env

---
 .github/workflows/checks.yml | 8 +++-----
 tox.ini                      | 8 ++++++++
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index a708474ef..e515959ad 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -25,6 +25,9 @@ jobs:
         - python-version: "3.10"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs
+        - python-version: "3.10"
+          env:
+            TOXENV: twinecheck
 
     steps:
     - uses: actions/checkout@v2
@@ -39,8 +42,3 @@ jobs:
       run: |
         pip install -U tox
         tox
-    - name: Twine check
-      run: |
-        pip install twine
-        python setup.py sdist
-        twine check dist/*
diff --git a/tox.ini b/tox.ini
index 2bf9454d0..2d94ba78f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -71,6 +71,14 @@ deps =
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 
+[testenv:twinecheck]
+basepython = python3
+deps =
+    twine==4.0.1
+commands =
+    python setup.py sdist
+    twine check dist/*
+
 [pinned]
 deps =
     cryptography==3.3

From 80194f1c0374b2aa429de4c7ea70a683c946db95 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 2 Oct 2022 15:22:06 -0300
Subject: [PATCH 668/716] CrawlSpider: add support for async def callbacks

---
 scrapy/spiders/crawl.py |  6 ++++--
 tests/spiders.py        | 12 ++++++++++++
 tests/test_crawl.py     | 11 +++++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 1dcf2e6ab..d860ae0b4 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -6,7 +6,7 @@ See documentation in docs/topics/spiders.rst
 """
 
 import copy
-from typing import Sequence
+from typing import Awaitable, Sequence
 
 from scrapy.http import Request, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
@@ -109,9 +109,11 @@ class CrawlSpider(Spider):
         rule = self._rules[failure.request.meta['rule']]
         return self._handle_failure(failure, rule.errback)
 
-    def _parse_response(self, response, callback, cb_kwargs, follow=True):
+    async def _parse_response(self, response, callback, cb_kwargs, follow=True):
         if callback:
             cb_res = callback(response, **cb_kwargs) or ()
+            if isinstance(cb_res, Awaitable):
+                cb_res = await cb_res
             cb_res = self.process_results(response, cb_res)
             for request_or_item in iterate_spider_output(cb_res):
                 yield request_or_item
diff --git a/tests/spiders.py b/tests/spiders.py
index 3b69aa7ae..2b78e1f7c 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -369,6 +369,18 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
         yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"})
 
 
+class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod):
+    """A CrawlSpider with an async def callback"""
+    name = 'crawl_spider_with_async_callback'
+    rules = (
+        Rule(LinkExtractor(), callback='parse_async', follow=True),
+    )
+
+    async def parse_async(self, response, foo=None):
+        self.logger.info('[parse_async] status %i (foo: %s)', response.status, foo)
+        return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"})
+
+
 class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
     name = 'crawl_spider_with_errback'
     rules = (
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index c11871745..d14021319 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -37,6 +37,7 @@ from tests.spiders import (
     BrokenStartRequestsSpider,
     BytesReceivedCallbackSpider,
     BytesReceivedErrbackSpider,
+    CrawlSpiderWithAsyncCallback,
     CrawlSpiderWithErrback,
     CrawlSpiderWithParseMethod,
     DelaySpider,
@@ -391,6 +392,16 @@ class CrawlSpiderTestCase(TestCase):
         self.assertIn("[parse] status 201 (foo: None)", str(log))
         self.assertIn("[parse] status 202 (foo: bar)", str(log))
 
+    @defer.inlineCallbacks
+    def test_crawlspider_with_async_callback(self):
+        crawler = get_crawler(CrawlSpiderWithAsyncCallback)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
+
+        self.assertIn("[parse_async] status 200 (foo: None)", str(log))
+        self.assertIn("[parse_async] status 201 (foo: None)", str(log))
+        self.assertIn("[parse_async] status 202 (foo: bar)", str(log))
+
     @defer.inlineCallbacks
     def test_crawlspider_with_errback(self):
         crawler = get_crawler(CrawlSpiderWithErrback)

From da8f915091a769b3c3aca0f60d2370c2240213a1 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 2 Oct 2022 17:37:10 -0300
Subject: [PATCH 669/716] Adapt for asyng generator callbacks

---
 scrapy/spiders/crawl.py  | 11 +++++++----
 scrapy/utils/asyncgen.py |  2 +-
 tests/spiders.py         | 12 ++++++++++++
 tests/test_crawl.py      | 11 +++++++++++
 4 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index d860ae0b4..edac082d0 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -6,11 +6,12 @@ See documentation in docs/topics/spiders.rst
 """
 
 import copy
-from typing import Awaitable, Sequence
+from typing import AsyncIterable, Awaitable, Sequence
 
-from scrapy.http import Request, HtmlResponse
+from scrapy.http import Request, Response, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
+from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.spider import iterate_spider_output
 
 
@@ -78,7 +79,7 @@ class CrawlSpider(Spider):
     def parse_start_url(self, response, **kwargs):
         return []
 
-    def process_results(self, response, results):
+    def process_results(self, response: Response, results: list):
         return results
 
     def _build_request(self, rule_index, link):
@@ -112,7 +113,9 @@ class CrawlSpider(Spider):
     async def _parse_response(self, response, callback, cb_kwargs, follow=True):
         if callback:
             cb_res = callback(response, **cb_kwargs) or ()
-            if isinstance(cb_res, Awaitable):
+            if isinstance(cb_res, AsyncIterable):
+                cb_res = await collect_asyncgen(cb_res)
+            elif isinstance(cb_res, Awaitable):
                 cb_res = await cb_res
             cb_res = self.process_results(response, cb_res)
             for request_or_item in iterate_spider_output(cb_res):
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 9f794de92..c84b51e8c 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,7 +1,7 @@
 from typing import AsyncGenerator, AsyncIterable, Iterable, Union
 
 
-async def collect_asyncgen(result: AsyncIterable):
+async def collect_asyncgen(result: AsyncIterable) -> list:
     results = []
     async for x in result:
         results.append(x)
diff --git a/tests/spiders.py b/tests/spiders.py
index 2b78e1f7c..5ea8a4a21 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -381,6 +381,18 @@ class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod):
         return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"})
 
 
+class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod):
+    """A CrawlSpider with an async generator callback"""
+    name = 'crawl_spider_with_async_generator_callback'
+    rules = (
+        Rule(LinkExtractor(), callback='parse_async_gen', follow=True),
+    )
+
+    async def parse_async_gen(self, response, foo=None):
+        self.logger.info('[parse_async_gen] status %i (foo: %s)', response.status, foo)
+        yield Request(self.mockserver.url("/status?n=202"), self.parse_async_gen, cb_kwargs={"foo": "bar"})
+
+
 class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
     name = 'crawl_spider_with_errback'
     rules = (
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index d14021319..8be4b6fe1 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -38,6 +38,7 @@ from tests.spiders import (
     BytesReceivedCallbackSpider,
     BytesReceivedErrbackSpider,
     CrawlSpiderWithAsyncCallback,
+    CrawlSpiderWithAsyncGeneratorCallback,
     CrawlSpiderWithErrback,
     CrawlSpiderWithParseMethod,
     DelaySpider,
@@ -402,6 +403,16 @@ class CrawlSpiderTestCase(TestCase):
         self.assertIn("[parse_async] status 201 (foo: None)", str(log))
         self.assertIn("[parse_async] status 202 (foo: bar)", str(log))
 
+    @defer.inlineCallbacks
+    def test_crawlspider_with_async_generator_callback(self):
+        crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
+
+        self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log))
+        self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log))
+        self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log))
+
     @defer.inlineCallbacks
     def test_crawlspider_with_errback(self):
         crawler = get_crawler(CrawlSpiderWithErrback)

From 41041ae740260c4bddda06ab7f4d8e4351e6e0e4 Mon Sep 17 00:00:00 2001
From: Felipe A <felipe.andrada@gmail.com>
Date: Mon, 3 Oct 2022 00:48:12 -0300
Subject: [PATCH 670/716] refact: add Osx DS_Store file to gitignore

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index d77d24624..6c5c50e08 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,3 +23,6 @@ test-output.*
 
 # Windows
 Thumbs.db
+
+# OSX miscellaneous
+.DS_Store
\ No newline at end of file

From 82d10f09145186fbd3a2c445287a7bf3e1e54263 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 6 Oct 2022 20:27:06 +0600
Subject: [PATCH 671/716] Add Ubuntu tests for Python 3.11rc2.

---
 .github/workflows/tests-ubuntu.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index be40c7c71..9e62b8e23 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -20,6 +20,12 @@ jobs:
         - python-version: "3.10"
           env:
             TOXENV: asyncio
+        - python-version: "3.11.0-rc.2"
+          env:
+            TOXENV: py
+        - python-version: "3.11.0-rc.2"
+          env:
+            TOXENV: asyncio
         - python-version: pypy3
           env:
             TOXENV: pypy3
@@ -53,7 +59,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install system libraries
-      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4'
+      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2'
       run: |
         sudo apt-get update
         # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version

From fa58ab21e4670db1b11b6097d809cd18dd2ca667 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 6 Oct 2022 20:59:06 +0600
Subject: [PATCH 672/716] Replace _getargspec_py23() with
 inspect.getfullargspec().

---
 scrapy/utils/python.py | 21 ++-------------------
 1 file changed, 2 insertions(+), 19 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 11c089ac2..8ce030d9d 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -180,23 +180,6 @@ def binary_is_text(data):
     return all(c not in _BINARYCHARS for c in data)
 
 
-def _getargspec_py23(func):
-    """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords,
-                                                        defaults)
-
-    Was identical to inspect.getargspec() in python2, but uses
-    inspect.getfullargspec() for python3 behind the scenes to avoid
-    DeprecationWarning.
-
-    >>> def f(a, b=2, *ar, **kw):
-    ...     pass
-
-    >>> _getargspec_py23(f)
-    ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,))
-    """
-    return inspect.ArgSpec(*inspect.getfullargspec(func)[:4])
-
-
 def get_func_args(func, stripself=False):
     """Return the argument name list of a callable"""
     if inspect.isfunction(func):
@@ -248,9 +231,9 @@ def get_spec(func):
     """
 
     if inspect.isfunction(func) or inspect.ismethod(func):
-        spec = _getargspec_py23(func)
+        spec = inspect.getfullargspec(func)
     elif hasattr(func, '__call__'):
-        spec = _getargspec_py23(func.__call__)
+        spec = inspect.getfullargspec(func.__call__)
     else:
         raise TypeError(f'{type(func)} is not callable')
 

From e60e8224a23ddd4fd98c54318d128f6744f40a6a Mon Sep 17 00:00:00 2001
From: Abinash Satapathy <iamabinash98@gmail.com>
Date: Thu, 6 Oct 2022 19:58:48 +0200
Subject: [PATCH 673/716] Update and rename INSTALL to INSTALL.md

---
 INSTALL    | 4 ----
 INSTALL.md | 4 ++++
 2 files changed, 4 insertions(+), 4 deletions(-)
 delete mode 100644 INSTALL
 create mode 100644 INSTALL.md

diff --git a/INSTALL b/INSTALL
deleted file mode 100644
index 06e812936..000000000
--- a/INSTALL
+++ /dev/null
@@ -1,4 +0,0 @@
-For information about installing Scrapy see:
-
-* docs/intro/install.rst (local file)
-* https://docs.scrapy.org/en/latest/intro/install.html (online version)
diff --git a/INSTALL.md b/INSTALL.md
new file mode 100644
index 000000000..495413f97
--- /dev/null
+++ b/INSTALL.md
@@ -0,0 +1,4 @@
+For information about installing Scrapy see:
+
+* [Local docs](docs/intro/install.rst)
+* [Online docs](https://docs.scrapy.org/en/latest/intro/install.html)

From 300c42bfdf0afe0809b03cf571ebe5da7352301e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 7 Oct 2022 14:53:19 +0600
Subject: [PATCH 674/716] Install PyPy using actions/setup-python.

---
 .github/workflows/tests-ubuntu.yml | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 9e62b8e23..633e01c95 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -26,10 +26,9 @@ jobs:
         - python-version: "3.11.0-rc.2"
           env:
             TOXENV: asyncio
-        - python-version: pypy3
+        - python-version: pypy3.9
           env:
             TOXENV: pypy3
-            PYPY_VERSION: 3.9-v7.3.9
 
         # pinned deps
         - python-version: 3.7.13
@@ -38,10 +37,9 @@ jobs:
         - python-version: 3.7.13
           env:
             TOXENV: asyncio-pinned
-        - python-version: pypy3
+        - python-version: pypy3.7
           env:
             TOXENV: pypy3-pinned
-            PYPY_VERSION: 3.7-v7.3.5
 
         # extras
         # extra-deps includes reppy, which does not support Python 3.9
@@ -59,7 +57,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install system libraries
-      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2'
+      if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2'
       run: |
         sudo apt-get update
         # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
@@ -68,13 +66,6 @@ jobs:
     - name: Run tests
       env: ${{ matrix.env }}
       run: |
-        if [[ ! -z "$PYPY_VERSION" ]]; then
-          export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
-          wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
-          tar -jxf ${PYPY_VERSION}.tar.bz2
-          $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
-          source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-        fi
         pip install -U tox
         tox
 

From 4aea925714f7c9beb68d3d478ad528f00d488942 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 7 Oct 2022 15:01:50 +0600
Subject: [PATCH 675/716] Update action versions.

---
 .github/workflows/tests-ubuntu.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 633e01c95..9cd0df468 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -49,10 +49,10 @@ jobs:
             TOXENV: extra-deps
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 

From 424849b27582d3a74fab0c43d842576722b48e25 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 7 Oct 2022 15:17:55 +0600
Subject: [PATCH 676/716] Update mypy.

---
 tox.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 2804ebe95..12a4516ce 100644
--- a/tox.ini
+++ b/tox.ini
@@ -38,7 +38,8 @@ install_command =
 basepython = python3
 deps =
     lxml-stubs==0.2.0
-    mypy==0.971
+    mypy==0.982
+    types-attrs==19.1.0
     types-pyOpenSSL==21.0.0
     types-setuptools==57.0.0
 commands =

From ccb6a8c098501c0f4322faeac17f9b97dad0c5e3 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 7 Oct 2022 15:27:14 +0600
Subject: [PATCH 677/716] Add a note about flake8.

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 12a4516ce..7002c2f18 100644
--- a/tox.ini
+++ b/tox.ini
@@ -59,6 +59,7 @@ deps =
     # Twisted[http2] is required to import some files
     Twisted[http2]>=17.9.0
     pytest-flake8==1.1.1
+    # newer ones don't work: https://github.com/tholo/pytest-flake8/issues/87
     flake8==4.0.1
 commands =
     pytest --flake8 {posargs:docs scrapy tests}

From d1515cc0755ae1d587ffbe4171544442fa10ef8d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 7 Oct 2022 15:30:44 +0600
Subject: [PATCH 678/716] Update pylint.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 7002c2f18..463828e7c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -69,7 +69,7 @@ commands =
 basepython = python3.8
 deps =
     {[testenv:extra-deps]deps}
-    pylint==2.14.5
+    pylint==2.15.3
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 

From ed9bc84d551c7302cd075aa52f30d38465749f05 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 8 Oct 2022 19:11:16 +0600
Subject: [PATCH 679/716] Remove a pin on libxml2-dev.

---
 .github/workflows/tests-ubuntu.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 9cd0df468..7915a9aab 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -60,8 +60,7 @@ jobs:
       if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2'
       run: |
         sudo apt-get update
-        # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
-        sudo apt-get install libxml2-dev/bionic-updates libxslt-dev
+        sudo apt-get install libxml2-dev libxslt-dev
 
     - name: Run tests
       env: ${{ matrix.env }}

From 92f2d75ed39e4d75f6cca23b8d3aaf934677f4f3 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 8 Oct 2022 19:12:32 +0600
Subject: [PATCH 680/716] Add a classifier for Python 3.11.

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 8b9c43738..a43cf08c8 100644
--- a/setup.py
+++ b/setup.py
@@ -83,6 +83,7 @@ setup(
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Internet :: WWW/HTTP',

From 5fa613b419f0d94b7dca8ae4ba2782ca268a9d7c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 10 Oct 2022 11:02:57 +0600
Subject: [PATCH 681/716] Run flake8 directly.

---
 conftest.py | 10 ----------
 tox.ini     |  3 +--
 2 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/conftest.py b/conftest.py
index 117087790..d7fe80321 100644
--- a/conftest.py
+++ b/conftest.py
@@ -42,16 +42,6 @@ def chdir(tmpdir):
     tmpdir.chdir()
 
 
-def pytest_collection_modifyitems(session, config, items):
-    # Avoid executing tests when executing `--flake8` flag (pytest-flake8)
-    try:
-        from pytest_flake8 import Flake8Item
-        if config.getoption('--flake8'):
-            items[:] = [item for item in items if isinstance(item, Flake8Item)]
-    except ImportError:
-        pass
-
-
 def pytest_addoption(parser):
     parser.addoption(
         "--reactor",
diff --git a/tox.ini b/tox.ini
index 463828e7c..822e96fde 100644
--- a/tox.ini
+++ b/tox.ini
@@ -58,11 +58,10 @@ deps =
     {[testenv]deps}
     # Twisted[http2] is required to import some files
     Twisted[http2]>=17.9.0
-    pytest-flake8==1.1.1
     # newer ones don't work: https://github.com/tholo/pytest-flake8/issues/87
     flake8==4.0.1
 commands =
-    pytest --flake8 {posargs:docs scrapy tests}
+    flake8 {posargs:docs scrapy tests}
 
 [testenv:pylint]
 # reppy does not support Python 3.9+

From eeb199adda2d7b56c78879df79c7294148ca15f1 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 10 Oct 2022 11:10:59 +0600
Subject: [PATCH 682/716] Fix flake8 issues in previously ignored files.

---
 .flake8                                       |  2 ++
 docs/_ext/scrapydocs.py                       | 24 +++++++++----------
 scrapy/utils/testsite.py                      |  2 +-
 .../CrawlerProcess/asyncio_deferred_signal.py |  1 -
 .../CrawlerProcess/asyncio_enabled_reactor.py |  4 ++--
 tests/CrawlerProcess/reactor_default.py       |  3 +--
 .../reactor_default_twisted_reactor_select.py |  4 +---
 tests/CrawlerProcess/reactor_select.py        |  2 --
 ..._select_subclass_twisted_reactor_select.py |  4 ----
 .../reactor_select_twisted_reactor_select.py  |  3 ---
 10 files changed, 19 insertions(+), 30 deletions(-)

diff --git a/.flake8 b/.flake8
index 1c503fb0b..d7aebc24b 100644
--- a/.flake8
+++ b/.flake8
@@ -4,6 +4,8 @@ max-line-length = 119
 ignore = W503
 
 exclude =
+    docs/conf.py
+
 # Exclude files that are meant to provide top-level imports
 # E402: Module level import not at top of file
 # F401: Module imported but unused
diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 640660943..d02a2e17b 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -80,24 +80,24 @@ def replace_settingslist_nodes(app, doctree, fromdocname):
 
 def setup(app):
     app.add_crossref_type(
-        directivename = "setting",
-        rolename      = "setting",
-        indextemplate = "pair: %s; setting",
+        directivename="setting",
+        rolename="setting",
+        indextemplate="pair: %s; setting",
     )
     app.add_crossref_type(
-        directivename = "signal",
-        rolename      = "signal",
-        indextemplate = "pair: %s; signal",
+        directivename="signal",
+        rolename="signal",
+        indextemplate="pair: %s; signal",
     )
     app.add_crossref_type(
-        directivename = "command",
-        rolename      = "command",
-        indextemplate = "pair: %s; command",
+        directivename="command",
+        rolename="command",
+        indextemplate="pair: %s; command",
     )
     app.add_crossref_type(
-        directivename = "reqmeta",
-        rolename      = "reqmeta",
-        indextemplate = "pair: %s; reqmeta",
+        directivename="reqmeta",
+        rolename="reqmeta",
+        indextemplate="pair: %s; reqmeta",
     )
     app.add_role('source', source_role)
     app.add_role('commit', commit_role)
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index fce77be32..5d3710391 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -23,7 +23,7 @@ class NoMetaRefreshRedirect(util.Redirect):
     def render(self, request):
         content = util.Redirect.render(self, request)
         return content.replace(b'http-equiv=\"refresh\"',
-            b'http-no-equiv=\"do-not-refresh-me\"')
+                               b'http-no-equiv=\"do-not-refresh-me\"')
 
 
 def test_site():
diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py
index bdd3c1fef..b83f6a585 100644
--- a/tests/CrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py
@@ -5,7 +5,6 @@ from typing import Optional
 from scrapy import Spider
 from scrapy.crawler import CrawlerProcess
 from scrapy.utils.defer import deferred_from_coro
-from twisted.internet.defer import Deferred
 
 
 class UppercasePipeline:
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py
index f2a93074b..e561d63c7 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@@ -6,8 +6,8 @@ if sys.version_info >= (3, 8) and sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncioreactor.install(asyncio.get_event_loop())
 
-import scrapy
-from scrapy.crawler import CrawlerProcess
+import scrapy  # noqa: E402
+from scrapy.crawler import CrawlerProcess  # noqa: E402
 
 
 class NoRequestsSpider(scrapy.Spider):
diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py
index 5a21a3717..2c867df61 100644
--- a/tests/CrawlerProcess/reactor_default.py
+++ b/tests/CrawlerProcess/reactor_default.py
@@ -1,6 +1,6 @@
 import scrapy
 from scrapy.crawler import CrawlerProcess
-from twisted.internet import reactor
+from twisted.internet import reactor  # noqa: F401
 
 
 class NoRequestsSpider(scrapy.Spider):
@@ -14,4 +14,3 @@ process = CrawlerProcess(settings={})
 
 process.crawl(NoRequestsSpider)
 process.start()
-
diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
index c476722ef..c2b30b044 100644
--- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
+++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
@@ -1,6 +1,6 @@
 import scrapy
 from scrapy.crawler import CrawlerProcess
-from twisted.internet import reactor
+from twisted.internet import reactor  # noqa: F401
 
 
 class NoRequestsSpider(scrapy.Spider):
@@ -16,5 +16,3 @@ process = CrawlerProcess(settings={
 
 process.crawl(NoRequestsSpider)
 process.start()
-
-
diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py
index eac6e2f89..ca70c06a0 100644
--- a/tests/CrawlerProcess/reactor_select.py
+++ b/tests/CrawlerProcess/reactor_select.py
@@ -15,5 +15,3 @@ process = CrawlerProcess(settings={})
 
 process.crawl(NoRequestsSpider)
 process.start()
-
-
diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
index 47f480605..0035daf1e 100644
--- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
+++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
@@ -25,7 +25,3 @@ process = CrawlerProcess(settings={
 
 process.crawl(NoRequestsSpider)
 process.start()
-
-
-
-
diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
index e0d2dab26..4f8394edb 100644
--- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
+++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
@@ -17,6 +17,3 @@ process = CrawlerProcess(settings={
 
 process.crawl(NoRequestsSpider)
 process.start()
-
-
-

From 5bf42606792c69268cc34da287293e9406e25883 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 10 Oct 2022 11:14:20 +0600
Subject: [PATCH 683/716] Update flake8.

---
 tests/test_http_response.py | 6 ++++--
 tox.ini                     | 3 +--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 5d67a5e74..b42c95045 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -705,7 +705,8 @@ class HtmlResponseTest(TextResponseTest):
 
     def test_html_encoding(self):
 
-        body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+        body = b"""<html><head><title>Some page</title>
+        <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
         </head><body>Price: \xa3100</body></html>'
         """
         r1 = self.response_class("http://www.example.com", body=body)
@@ -719,7 +720,8 @@ class HtmlResponseTest(TextResponseTest):
         self._assert_response_values(r2, 'iso-8859-1', body)
 
         # for conflicting declarations headers must take precedence
-        body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+        body = b"""<html><head><title>Some page</title>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
         </head><body>Price: \xa3100</body></html>'
         """
         r3 = self.response_class("http://www.example.com", body=body,
diff --git a/tox.ini b/tox.ini
index 822e96fde..eee99cb2d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -58,8 +58,7 @@ deps =
     {[testenv]deps}
     # Twisted[http2] is required to import some files
     Twisted[http2]>=17.9.0
-    # newer ones don't work: https://github.com/tholo/pytest-flake8/issues/87
-    flake8==4.0.1
+    flake8==5.0.4
 commands =
     flake8 {posargs:docs scrapy tests}
 

From b792632046093504b1df8bcb0b92d4cb52ebf436 Mon Sep 17 00:00:00 2001
From: Nirjas Jakilim <nirjas01@student.sust.edu>
Date: Mon, 10 Oct 2022 11:47:02 +0600
Subject: [PATCH 684/716] updated setup-python and checkout workflow

---
 .github/workflows/checks.yml  | 4 ++--
 .github/workflows/publish.yml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index e515959ad..cc8f20f44 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -30,10 +30,10 @@ jobs:
             TOXENV: twinecheck
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 44b682830..8e307189d 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -7,10 +7,10 @@ jobs:
     if: startsWith(github.event.ref, 'refs/tags/')
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: "3.10"
 

From d12fcc555b57dbd62c90212bd1d918d39e1ba45d Mon Sep 17 00:00:00 2001
From: Derek <dschaller@users.noreply.github.com>
Date: Tue, 11 Oct 2022 10:32:45 -0700
Subject: [PATCH 685/716] Link to the Code of Conduct (#5659)

---
 README.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 358302c76..d416ced3c 100644
--- a/README.rst
+++ b/README.rst
@@ -95,8 +95,7 @@ See https://docs.scrapy.org/en/master/contributing.html for details.
 Code of Conduct
 ---------------
 
-Please note that this project is released with a Contributor Code of Conduct
-(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
+Please note that this project is released with a Contributor `Code of Conduct <https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md>`_.
 
 By participating in this project you agree to abide by its terms.
 Please report unacceptable behavior to opensource@zyte.com.

From 96fb663ae1a95286b0634bc51bc17b42eb29bd0f Mon Sep 17 00:00:00 2001
From: Abdul Rauf <abdulraufmujahid@gmail.com>
Date: Tue, 11 Oct 2022 10:34:18 -0700
Subject: [PATCH 686/716] README: set Bash highlighting for pip install (#5648)

---
 README.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index d416ced3c..970bf2c35 100644
--- a/README.rst
+++ b/README.rst
@@ -64,7 +64,9 @@ Requirements
 Install
 =======
 
-The quick way::
+The quick way:
+
+.. code:: bash
 
     pip install scrapy
 

From da9a2f8a946d6341be1cdd8cad0616a18bba6dbe Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Wed, 12 Oct 2022 11:10:39 -0300
Subject: [PATCH 687/716] Remove mention of minimum PyPy versions from the
 documentation (#5678)

---
 docs/intro/install.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 80a9c16d6..f28f5216a 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -10,7 +10,7 @@ Supported Python versions
 =========================
 
 Scrapy requires Python 3.7+, either the CPython implementation (default) or
-the PyPy 7.3.5+ implementation (see :ref:`python:implementations`).
+the PyPy implementation (see :ref:`python:implementations`).
 
 .. _intro-install-scrapy:
 
@@ -219,7 +219,7 @@ After any of these workarounds you should be able to install Scrapy::
 PyPy
 ----
 
-We recommend using the latest PyPy version. The version tested is 5.9.0.
+We recommend using the latest PyPy version.
 For PyPy3, only Linux installation was tested.
 
 Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.

From 715c05d504d22e87935ae42cee55ee35b12c2ebd Mon Sep 17 00:00:00 2001
From: gabrielztk <38334108+gabrielztk@users.noreply.github.com>
Date: Thu, 13 Oct 2022 07:22:10 -0300
Subject: [PATCH 688/716] =?UTF-8?q?transport.producer.loseConnection()=20?=
 =?UTF-8?q?=E2=86=92=20transport.loseConnection()=20(#4995)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/core/downloader/handlers/http11.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 38935667d..6b8a18f1a 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -384,8 +384,7 @@ class ScrapyAgent:
                 logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
                              {"request": request, "handler": handler.__qualname__})
                 txresponse._transport.stopProducing()
-                with suppress(AttributeError):
-                    txresponse._transport._producer.loseConnection()
+                txresponse._transport.loseConnection()
                 return {
                     "txresponse": txresponse,
                     "body": b"",
@@ -417,7 +416,7 @@ class ScrapyAgent:
 
             logger.warning(warning_msg, warning_args)
 
-            txresponse._transport._producer.loseConnection()
+            txresponse._transport.loseConnection()
             raise defer.CancelledError(warning_msg % warning_args)
 
         if warnsize and expected_size > warnsize:
@@ -543,7 +542,7 @@ class _ResponseReader(protocol.Protocol):
                 logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
                              {"request": self._request, "handler": handler.__qualname__})
                 self.transport.stopProducing()
-                self.transport._producer.loseConnection()
+                self.transport.loseConnection()
                 failure = result if result.value.fail else None
                 self._finish_response(flags=["download_stopped"], failure=failure)
 

From 62cc26e209b66ce5941f15736f669c75dcac9a59 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 14 Oct 2022 22:03:54 +0600
Subject: [PATCH 689/716] Change TWISTED_REACTOR in the default template.

---
 docs/topics/settings.rst                         | 5 +++++
 scrapy/templates/project/module/settings.py.tmpl | 1 +
 2 files changed, 6 insertions(+)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a711fd197..0b1ef71cf 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1642,6 +1642,11 @@ install the default reactor defined by Twisted for the current platform. This
 is to maintain backward compatibility and avoid possible problems caused by
 using a non-default reactor.
 
+.. versionchanged:: VERSION
+   The :command:`startproject` command now sets this setting to
+   ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated
+   ``settings.py`` file.
+
 For additional information, see :doc:`core/howto/choosing-reactor`.
 
 
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 5e541e2c0..c0c34e986 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -89,3 +89,4 @@ ROBOTSTXT_OBEY = True
 
 # Set settings whose default value is deprecated to a future-proof value
 REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION'
+TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'

From 22a59d0005c03d866ce98d352024974a9e48e7d1 Mon Sep 17 00:00:00 2001
From: Nirjas Jakilim <nirjas01@student.sust.edu>
Date: Fri, 14 Oct 2022 23:41:50 +0600
Subject: [PATCH 690/716] CI: use the latest version of Ubuntu (#5675)

---
 .github/workflows/checks.yml       | 2 +-
 .github/workflows/publish.yml      | 2 +-
 .github/workflows/tests-ubuntu.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index cc8f20f44..439dfee51 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -3,7 +3,7 @@ on: [push, pull_request]
 
 jobs:
   checks:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 8e307189d..f6b098b80 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -3,7 +3,7 @@ on: [push]
 
 jobs:
   publish:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     if: startsWith(github.event.ref, 'refs/tags/')
 
     steps:
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 7915a9aab..d2bfe4a5f 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -3,7 +3,7 @@ on: [push, pull_request]
 
 jobs:
   tests:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:

From 043575123c57db9f055c3668a3d80485989df0b2 Mon Sep 17 00:00:00 2001
From: Mohammadtaher Abbasi <mohammadtaher.abbasi@zoodfood.com>
Date: Sat, 15 Oct 2022 11:41:05 +0330
Subject: [PATCH 691/716] Add async callback support to the parse command
 (#5577)

---
 scrapy/commands/parse.py    | 55 ++++++++++++++++++++++---------------
 tests/test_command_parse.py | 15 ++++++++++
 2 files changed, 48 insertions(+), 22 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 9b4fb0ed6..d93ab2ac5 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -5,6 +5,8 @@ from typing import Dict
 from itemadapter import is_item, ItemAdapter
 from w3lib.url import is_url
 
+from twisted.internet.defer import maybeDeferred
+
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.http import Request
 from scrapy.utils import display
@@ -110,16 +112,19 @@ class Command(BaseRunSpiderCommand):
             if not opts.nolinks:
                 self.print_requests(colour=colour)
 
-    def run_callback(self, response, callback, cb_kwargs=None):
-        cb_kwargs = cb_kwargs or {}
+    def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):
         items, requests = [], []
-
-        for x in iterate_spider_output(callback(response, **cb_kwargs)):
+        for x in spider_output:
             if is_item(x):
                 items.append(x)
             elif isinstance(x, Request):
                 requests.append(x)
-        return items, requests
+        return items, requests, opts, depth, spider, callback
+
+    def run_callback(self, response, callback, cb_kwargs=None):
+        cb_kwargs = cb_kwargs or {}
+        d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))
+        return d
 
     def get_callback_from_rules(self, spider, response):
         if getattr(spider, 'rules', None):
@@ -158,6 +163,25 @@ class Command(BaseRunSpiderCommand):
             logger.error('No response downloaded for: %(url)s',
                          {'url': url})
 
+    def scraped_data(self, args):
+        items, requests, opts, depth, spider, callback = args
+        if opts.pipelines:
+            itemproc = self.pcrawler.engine.scraper.itemproc
+            for item in items:
+                itemproc.process_item(item, spider)
+        self.add_items(depth, items)
+        self.add_requests(depth, requests)
+
+        scraped_data = items if opts.output else []
+        if depth < opts.depth:
+            for req in requests:
+                req.meta['_depth'] = depth + 1
+                req.meta['_callback'] = req.callback
+                req.callback = callback
+            scraped_data += requests
+
+        return scraped_data
+
     def prepare_request(self, spider, request, opts):
         def callback(response, **cb_kwargs):
             # memorize first request
@@ -191,23 +215,10 @@ class Command(BaseRunSpiderCommand):
             # parse items and requests
             depth = response.meta['_depth']
 
-            items, requests = self.run_callback(response, cb, cb_kwargs)
-            if opts.pipelines:
-                itemproc = self.pcrawler.engine.scraper.itemproc
-                for item in items:
-                    itemproc.process_item(item, spider)
-            self.add_items(depth, items)
-            self.add_requests(depth, requests)
-
-            scraped_data = items if opts.output else []
-            if depth < opts.depth:
-                for req in requests:
-                    req.meta['_depth'] = depth + 1
-                    req.meta['_callback'] = req.callback
-                    req.callback = callback
-                scraped_data += requests
-
-            return scraped_data
+            d = self.run_callback(response, cb, cb_kwargs)
+            d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
+            d.addCallback(self.scraped_data)
+            return d
 
         # update request meta if any extra meta was passed through the --meta/-m opts.
         if opts.meta:
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 0d992be56..8368356e2 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -29,7 +29,15 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
 import scrapy
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule
+from scrapy.utils.test import get_from_asyncio_queue
 
+class AsyncDefAsyncioSpider(scrapy.Spider):
+
+    name = 'asyncdef{self.spider_name}'
+
+    async def parse(self, response):
+        status = await get_from_asyncio_queue(response.status)
+        return [scrapy.Item(), dict(foo='bar')]
 
 class MySpider(scrapy.Spider):
     name = '{self.spider_name}'
@@ -160,6 +168,13 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
                                            self.url('/html')])
         self.assertIn("INFO: It Works!", _textmode(stderr))
 
+    @defer.inlineCallbacks
+    def test_asyncio_parse_items(self):
+        status, out, stderr = yield self.execute(
+            ['--spider', 'asyncdef' + self.spider_name, '-c', 'parse', self.url('/html')]
+        )
+        self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
+
     @defer.inlineCallbacks
     def test_parse_items(self):
         status, out, stderr = yield self.execute(

From 75bb516edbc39f9a657e71e75f05f0fd6a33d60d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sat, 15 Oct 2022 10:26:38 +0200
Subject: [PATCH 692/716] Adapt tests to the new value of TWISTED_REACTOR for
 new projects

---
 tests/test_commands.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 76d5f3935..eaca41102 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -689,8 +689,15 @@ class MySpider(scrapy.Spider):
         ])
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    def test_asyncio_enabled_false(self):
+    def test_asyncio_enabled_default(self):
         log = self.get_log(self.debug_log_spider, args=[])
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+
+    def test_asyncio_enabled_false(self):
+        log = self.get_log(self.debug_log_spider, args=[
+            '-s', 'TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor'
+        ])
+        self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
     @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')

From 960a7f68f6939744b39d528e28f8f925a2e12ad0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Sat, 15 Oct 2022 11:27:00 +0200
Subject: [PATCH 693/716] Verify that the installed asyncio event loop matches
 ASYNCIO_EVENT_LOOP (#5529)

Co-authored-by: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
---
 scrapy/crawler.py                             | 14 ++++++++--
 scrapy/utils/reactor.py                       | 18 ++++++++++++
 .../asyncio_enabled_reactor_different_loop.py | 25 +++++++++++++++++
 .../asyncio_enabled_reactor_same_loop.py      | 28 +++++++++++++++++++
 tests/test_crawler.py                         | 23 +++++++++++++++
 5 files changed, 105 insertions(+), 3 deletions(-)
 create mode 100644 tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
 create mode 100644 tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index e768bca12..65174d846 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -31,7 +31,12 @@ from scrapy.utils.log import (
 )
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
-from scrapy.utils.reactor import install_reactor, verify_installed_reactor
+from scrapy.utils.reactor import (
+    install_reactor,
+    is_asyncio_reactor_installed,
+    verify_installed_asyncio_event_loop,
+    verify_installed_reactor,
+)
 
 
 logger = logging.getLogger(__name__)
@@ -78,17 +83,20 @@ class Crawler:
             crawler=self,
         )
 
-        reactor_class = self.settings.get("TWISTED_REACTOR")
+        reactor_class = self.settings["TWISTED_REACTOR"]
+        event_loop = self.settings["ASYNCIO_EVENT_LOOP"]
         if init_reactor:
             # this needs to be done after the spider settings are merged,
             # but before something imports twisted.internet.reactor
             if reactor_class:
-                install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
+                install_reactor(reactor_class, event_loop)
             else:
                 from twisted.internet import reactor  # noqa: F401
             log_reactor_info()
         if reactor_class:
             verify_installed_reactor(reactor_class)
+            if is_asyncio_reactor_installed() and event_loop:
+                verify_installed_asyncio_event_loop(event_loop)
 
         self.extensions = ExtensionManager.from_crawler(self)
 
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index bc543b230..652733ce8 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -90,6 +90,24 @@ def verify_installed_reactor(reactor_path):
         raise Exception(msg)
 
 
+def verify_installed_asyncio_event_loop(loop_path):
+    from twisted.internet import reactor
+    loop_class = load_object(loop_path)
+    if isinstance(reactor._asyncioEventloop, loop_class):
+        return
+    installed = (
+        f"{reactor._asyncioEventloop.__class__.__module__}"
+        f".{reactor._asyncioEventloop.__class__.__qualname__}"
+    )
+    specified = f"{loop_class.__module__}.{loop_class.__qualname__}"
+    raise Exception(
+        "Scrapy found an asyncio Twisted reactor already "
+        f"installed, and its event loop class ({installed}) does "
+        "not match the one specified in the ASYNCIO_EVENT_LOOP "
+        f"setting ({specified})"
+    )
+
+
 def is_asyncio_reactor_installed():
     from twisted.internet import reactor
     return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
new file mode 100644
index 000000000..ea8242f67
--- /dev/null
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
@@ -0,0 +1,25 @@
+import asyncio
+import sys
+
+from twisted.internet import asyncioreactor
+if sys.version_info >= (3, 8) and sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+asyncioreactor.install(asyncio.get_event_loop())
+
+import scrapy  # noqa: E402
+from scrapy.crawler import CrawlerProcess  # noqa: E402
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
+})
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
new file mode 100644
index 000000000..d24bf3031
--- /dev/null
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
@@ -0,0 +1,28 @@
+import asyncio
+import sys
+
+from uvloop import Loop
+
+from twisted.internet import asyncioreactor
+if sys.version_info >= (3, 8) and sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+asyncio.set_event_loop(Loop())
+asyncioreactor.install(asyncio.get_event_loop())
+
+import scrapy  # noqa: E402
+from scrapy.crawler import CrawlerProcess  # noqa: E402
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
+})
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index cf15ba9b9..c61d461f7 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -454,6 +454,29 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
         self.assertIn("async pipeline opened!", log)
 
+    @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
+    @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
+    def test_asyncio_enabled_reactor_same_loop(self):
+        log = self.run_script("asyncio_enabled_reactor_same_loop.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn("Using asyncio event loop: uvloop.Loop", log)
+
+    @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
+    @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
+    def test_asyncio_enabled_reactor_different_loop(self):
+        log = self.run_script("asyncio_enabled_reactor_different_loop.py")
+        self.assertNotIn("Spider closed (finished)", log)
+        self.assertIn(
+            (
+                "does not match the one specified in the ASYNCIO_EVENT_LOOP "
+                "setting (uvloop.Loop)"
+            ),
+            log,
+        )
+
     def test_default_loop_asyncio_deferred_signal(self):
         log = self.run_script("asyncio_deferred_signal.py")
         self.assertIn("Spider closed (finished)", log)

From c49764ffd7111d8a465a0d077d0df38bd40449fa Mon Sep 17 00:00:00 2001
From: mattkohl-flex <matthew.kohl@indeedflex.com>
Date: Mon, 17 Oct 2022 11:15:17 +0100
Subject: [PATCH 694/716] typo fixes

---
 docs/intro/install.rst           | 2 +-
 docs/topics/contracts.rst        | 2 +-
 docs/topics/extensions.rst       | 2 +-
 docs/topics/leaks.rst            | 2 +-
 docs/topics/request-response.rst | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index f28f5216a..9ab479edd 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -187,7 +187,7 @@ solutions:
   * Install `homebrew`_ following the instructions in https://brew.sh/
 
   * Update your ``PATH`` variable to state that homebrew packages should be
-    used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
+    used before system packages (Change ``.bashrc`` to ``.zshrc`` accordingly
     if you're using `zsh`_ as default shell)::
 
       echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index ef296dc9e..c29a3a410 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -102,7 +102,7 @@ override three methods:
     .. method:: Contract.post_process(output)
 
         This allows processing the output of the callback. Iterators are
-        converted listified before being passed to this hook.
+        converted to lists before being passed to this hook.
 
 Raise :class:`~scrapy.exceptions.ContractFail` from
 :class:`~scrapy.contracts.Contract.pre_process` or
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 297e1fdc5..130657b0b 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -17,7 +17,7 @@ settings, just like any other Scrapy code.
 
 It is customary for extensions to prefix their settings with their own name, to
 avoid collision with existing (and future) extensions. For example, a
-hypothetic extension to handle `Google Sitemaps`_ would use settings like
+hypothetical extension to handle `Google Sitemaps`_ would use settings like
 ``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on.
 
 .. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 477652704..33441838a 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -154,7 +154,7 @@ Too many spiders?
 If your project has too many spiders executed in parallel,
 the output of :func:`prefs()` can be difficult to read.
 For this reason, that function has a ``ignore`` argument which can be used to
-ignore a particular class (and all its subclases). For
+ignore a particular class (and all its subclasses). For
 example, this won't show any live references to spiders:
 
 >>> from scrapy.spiders import Spider
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 49cb69f67..7eb6942ac 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -446,7 +446,7 @@ class).
 Scenarios where changing the request fingerprinting algorithm may cause
 undesired results include, for example, using the HTTP cache middleware (see
 :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
-Changing the request fingerprinting algorithm would invalidade the current
+Changing the request fingerprinting algorithm would invalidate the current
 cache, requiring you to redownload all requests again.
 
 Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in

From 06c8f673afe9af08784e62d67930a8cbc9887ced Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 17 Oct 2022 15:04:29 +0200
Subject: [PATCH 695/716] 2.7 release notes (#5680)

* Fix the display name of documented fingerprinter class methods

* Initial draft for the Scrapy 2.7 release notes

* Update VERSION and PREVIOUS_VERSION references

* Clarify the restrictions lifted for item field output names

* Fix the description of the BOM bug fix

* Fix the note about changes in MIME sniffing

* Fix typo

* Extend highlights

* Fyx typo
---
 docs/_ext/scrapydocs.py                       |   2 +-
 docs/news.rst                                 | 193 +++++++++++++++++-
 docs/topics/components.rst                    |   4 +-
 docs/topics/coroutines.rst                    |  12 +-
 docs/topics/request-response.rst              |  45 ++--
 docs/topics/settings.rst                      |  10 +-
 docs/topics/spider-middleware.rst             |   8 +-
 scrapy/settings/default_settings.py           |   2 +-
 .../templates/project/module/settings.py.tmpl |   2 +-
 scrapy/utils/request.py                       |  12 +-
 scrapy/utils/test.py                          |   2 +-
 tests/test_crawl.py                           |   2 +-
 tests/test_crawler.py                         |   8 +-
 tests/test_dupefilters.py                     |  12 +-
 tests/test_pipeline_crawl.py                  |   2 +-
 tests/test_scheduler.py                       |   2 +-
 tests/test_spiderloader/__init__.py           |   2 +-
 tests/test_utils_request.py                   |   4 +-
 18 files changed, 259 insertions(+), 65 deletions(-)

diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index d02a2e17b..f0f382da3 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -15,7 +15,7 @@ class SettingsListDirective(Directive):
 
 
 def is_setting_index(node):
-    if node.tagname == 'index':
+    if node.tagname == 'index' and node['entries']:
         # index entries for setting directives look like:
         # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
         entry_type, info, refid = node['entries'][0][:3]
diff --git a/docs/news.rst b/docs/news.rst
index 9469d0fe5..d8b9fcd1e 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,195 @@
 Release notes
 =============
 
+.. _release-2.7.0:
+
+Scrapy 2.7.0 (to be released)
+-----------------------------
+
+Highlights:
+
+-   Added Python 3.11 support, dropped Python 3.6 support
+-   Improved support for :ref:`asynchronous callbacks <topics-coroutines>`
+-   :ref:`Asyncio support <using-asyncio>` is enabled by default on new
+    projects
+-   Output names of item fields can now be arbitrary strings
+-   Centralized :ref:`request fingerprinting <request-fingerprints>`
+    configuration is now possible
+
+Modified requirements
+~~~~~~~~~~~~~~~~~~~~~
+
+Python 3.7 or greater is now required; support for Python 3.6 has been dropped.
+Support for the upcoming Python 3.11 has been added.
+
+The minimum required version of some dependencies has changed as well:
+
+-   lxml_: 3.5.0 → 4.3.0
+
+-   Pillow_ (:ref:`images pipeline <images-pipeline>`): 4.0.0 → 7.1.0
+
+-   zope.interface_: 5.0.0 → 5.1.0
+
+(:issue:`5512`, :issue:`5514`, :issue:`5524`, :issue:`5563`, :issue:`5664`,
+:issue:`5670`, :issue:`5678`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+-   :meth:`ImagesPipeline.thumb_path
+    <scrapy.pipelines.images.ImagesPipeline.thumb_path>` must now accept an
+    ``item`` parameter (:issue:`5504`, :issue:`5508`).
+
+-   The ``scrapy.downloadermiddlewares.decompression`` module is now
+    deprecated (:issue:`5546`, :issue:`5547`).
+
+
+New features
+~~~~~~~~~~~~
+
+-   The
+    :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+    method of :ref:`spider middlewares <topics-spider-middleware>` can now be
+    defined as an :term:`asynchronous generator` (:issue:`4978`).
+
+-   The output of :class:`~scrapy.Request` callbacks defined as
+    :ref:`coroutines <topics-coroutines>` is now processed asynchronously
+    (:issue:`4978`).
+
+-   :class:`~scrapy.spiders.crawl.CrawlSpider` now supports :ref:`asynchronous
+    callbacks <topics-coroutines>` (:issue:`5657`).
+
+-   New projects created with the :command:`startproject` command have
+    :ref:`asyncio support <using-asyncio>` enabled by default (:issue:`5590`,
+    :issue:`5679`).
+
+-   The :setting:`FEED_EXPORT_FIELDS` setting can now be defined as a
+    dictionary to customize the output name of item fields, lifting the
+    restriction that required output names to be valid Python identifiers, e.g.
+    preventing them to have whitespace (:issue:`1008`, :issue:`3266`,
+    :issue:`3696`).
+
+-   You can now customize :ref:`request fingerprinting <request-fingerprints>`
+    through the new :setting:`REQUEST_FINGERPRINTER_CLASS` setting, instead of
+    having to change it on every Scrapy component that relies on request
+    fingerprinting (:issue:`900`, :issue:`3420`, :issue:`4113`, :issue:`4762`,
+    :issue:`4524`).
+
+-   ``jsonl`` is now supported and encouraged as a file extension for `JSON
+    Lines`_ files (:issue:`4848`).
+
+    .. _JSON Lines: https://jsonlines.org/
+
+-   :meth:`ImagesPipeline.thumb_path
+    <scrapy.pipelines.images.ImagesPipeline.thumb_path>` now receives the
+    source :ref:`item <topics-items>` (:issue:`5504`, :issue:`5508`).
+
+
+Bug fixes
+~~~~~~~~~
+
+-   When using Google Cloud Storage with a :ref:`media pipeline
+    <topics-media-pipeline>`, :setting:`FILES_EXPIRES` now also works when
+    :setting:`FILES_STORE` does not point at the root of your Google Cloud
+    Storage bucket (:issue:`5317`, :issue:`5318`).
+
+-   The :command:`parse` command now supports :ref:`asynchronous callbacks
+    <topics-coroutines>` (:issue:`5424`, :issue:`5577`).
+
+-   When using the :command:`parse` command with a URL for which there is no
+    available spider, an exception is no longer raised (:issue:`3264`,
+    :issue:`3265`, :issue:`5375`, :issue:`5376`, :issue:`5497`).
+
+-   :class:`~scrapy.http.TextResponse` now gives higher priority to the `byte
+    order mark`_ when determining the text encoding of the response body,
+    following the `HTML living standard`_ (:issue:`5601`, :issue:`5611`).
+
+    .. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark
+    .. _HTML living standard: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
+
+-   MIME sniffing takes the response body into account in FTP and HTTP/1.0
+    requests, as well as in cached requests (:issue:`4873`).
+
+-   MIME sniffing now detects valid HTML 5 documents even if the ``html`` tag
+    is missing (:issue:`4873`).
+
+-   An exception is now raised if :setting:`ASYNCIO_EVENT_LOOP` has a value
+    that does not match the asyncio event loop actually installed
+    (:issue:`5529`).
+
+-   Fixed :meth:`Headers.getlist <scrapy.http.headers.Headers.getlist>`
+    returning only the last header (:issue:`5515`, :issue:`5526`).
+
+-   Fixed :class:`LinkExtractor
+    <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` not ignoring the
+    ``tar.gz`` file extension by default (:issue:`1837`, :issue:`2067`,
+    :issue:`4066`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+-   Clarified the return type of :meth:`Spider.parse <scrapy.Spider.parse>`
+    (:issue:`5602`, :issue:`5608`).
+
+-   To enable
+    :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
+    to do `brotli compression`_, installing brotli_ is now recommended instead
+    of installing brotlipy_, as the former provides a more recent version of
+    brotli.
+
+    .. _brotli: https://github.com/google/brotli
+    .. _brotli compression: https://www.ietf.org/rfc/rfc7932.txt
+
+-   :ref:`Signal documentation <topics-signals>` now mentions :ref:`coroutine
+    support <topics-coroutines>` and uses it in code examples (:issue:`4852`,
+    :issue:`5358`).
+
+-   :ref:`bans` now recommends `Common Crawl`_ instead of `Google cache`_
+    (:issue:`3582`, :issue:`5432`).
+
+    .. _Common Crawl: https://commoncrawl.org/
+    .. _Google cache: http://www.googleguide.com/cached_pages.html
+
+-   The new :ref:`topics-components` topic covers enforcing requirements on
+    Scrapy components, like :ref:`downloader middlewares
+    <topics-downloader-middleware>`, :ref:`extensions <topics-extensions>`,
+    :ref:`item pipelines <topics-item-pipeline>`, :ref:`spider middlewares
+    <topics-spider-middleware>`, and more; :ref:`enforce-asyncio-requirement`
+    has also been added (:issue:`4978`).
+
+-   :ref:`topics-settings` now indicates that setting values must be
+    :ref:`picklable <pickle-picklable>` (:issue:`5607`, :issue:`5629`).
+
+-   Removed outdated documentation (:issue:`5446`, :issue:`5373`,
+    :issue:`5369`, :issue:`5370`, :issue:`5554`).
+
+-   Fixed typos (:issue:`5442`, :issue:`5455`, :issue:`5457`, :issue:`5461`,
+    :issue:`5538`, :issue:`5553`, :issue:`5558`, :issue:`5624`, :issue:`5631`).
+
+-   Fixed other issues (:issue:`5283`, :issue:`5284`, :issue:`5559`,
+    :issue:`5567`, :issue:`5648`, :issue:`5659`, :issue:`5665`).
+
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+-   Added a continuous integration job to run `twine check`_ (:issue:`5655`,
+    :issue:`5656`).
+
+    .. _twine check: https://twine.readthedocs.io/en/stable/#twine-check
+
+-   Addressed test issues and warnings (:issue:`5560`, :issue:`5561`,
+    :issue:`5612`, :issue:`5617`, :issue:`5639`, :issue:`5645`, :issue:`5662`,
+    :issue:`5671`, :issue:`5675`).
+
+-   Cleaned up code (:issue:`4991`, :issue:`4995`, :issue:`5451`,
+    :issue:`5487`, :issue:`5542`, :issue:`5667`, :issue:`5668`, :issue:`5672`).
+
+-   Applied minor code improvements (:issue:`5661`).
+
+
 .. _release-2.6.3:
 
 Scrapy 2.6.3 (2022-09-27)
@@ -3139,7 +3328,7 @@ New Features
 ~~~~~~~~~~~~
 
 - Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`)
-- Support `brotli`_-compressed content; requires optional `brotlipy`_
+- Support `brotli-compressed`_ content; requires optional `brotlipy`_
   (:issue:`2535`)
 - New :ref:`response.follow <response-follow-example>` shortcut
   for creating requests (:issue:`1940`)
@@ -3176,7 +3365,7 @@ New Features
 - ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command
   (:issue:`2740`)
 
-.. _brotli: https://github.com/google/brotli
+.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
 .. _brotlipy: https://github.com/python-hyper/brotlipy/
 
 Bug fixes
diff --git a/docs/topics/components.rst b/docs/topics/components.rst
index c44f3def2..ca301b827 100644
--- a/docs/topics/components.rst
+++ b/docs/topics/components.rst
@@ -75,9 +75,9 @@ If your requirement is a minimum Scrapy version, you may use
     class MyComponent:
 
         def __init__(self):
-            if parse_version(scrapy.__version__) < parse_version('VERSION'):
+            if parse_version(scrapy.__version__) < parse_version('2.7'):
                 raise RuntimeError(
-                    f"{MyComponent.__qualname__} requires Scrapy VERSION or "
+                    f"{MyComponent.__qualname__} requires Scrapy 2.7 or "
                     f"later, which allow defining the process_spider_output "
                     f"method of spider middlewares as an asynchronous "
                     f"generator."
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 750263385..a1ba4ba5c 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -22,7 +22,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
     If you are using any custom or third-party :ref:`spider middleware
     <topics-spider-middleware>`, see :ref:`sync-async-spider-middleware`.
 
-    .. versionchanged:: VERSION
+    .. versionchanged:: 2.7
        Output of async callbacks is now processed asynchronously instead of
        collecting all of it first.
 
@@ -49,7 +49,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
     See also :ref:`sync-async-spider-middleware` and
     :ref:`universal-spider-middleware`.
 
-    .. versionadded:: VERSION
+    .. versionadded:: 2.7
 
 General usage
 =============
@@ -129,7 +129,7 @@ Common use cases for asynchronous code include:
 Mixing synchronous and asynchronous spider middlewares
 ======================================================
 
-.. versionadded:: VERSION
+.. versionadded:: 2.7
 
 The output of a :class:`~scrapy.Request` callback is passed as the ``result``
 parameter to the
@@ -182,10 +182,10 @@ process_spider_output_async method <universal-spider-middleware>`.
 Universal spider middlewares
 ============================
 
-.. versionadded:: VERSION
+.. versionadded:: 2.7
 
 To allow writing a spider middleware that supports asynchronous execution of
-its ``process_spider_output`` method in Scrapy VERSION and later (avoiding
+its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding
 :ref:`asynchronous-to-synchronous conversions <sync-async-spider-middleware>`)
 while maintaining support for older Scrapy versions, you may define
 ``process_spider_output`` as a synchronous method and define an
@@ -206,7 +206,7 @@ For example::
                 yield r
 
 .. note:: This is an interim measure to allow, for a time, to write code that
-          works in Scrapy VERSION and later without requiring
+          works in Scrapy 2.7 and later without requiring
           asynchronous-to-synchronous conversions, and works in earlier Scrapy
           versions as well.
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 49cb69f67..4393e1c68 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -394,7 +394,7 @@ To change how request fingerprints are built for your requests, use the
 REQUEST_FINGERPRINTER_CLASS
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. versionadded:: VERSION
+.. versionadded:: 2.7
 
 Default: :class:`scrapy.utils.request.RequestFingerprinter`
 
@@ -409,38 +409,38 @@ import path.
 REQUEST_FINGERPRINTER_IMPLEMENTATION
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. versionadded:: VERSION
+.. versionadded:: 2.7
 
-Default: ``'PREVIOUS_VERSION'``
+Default: ``'2.6'``
 
 Determines which request fingerprinting algorithm is used by the default
 request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
 
 Possible values are:
 
--   ``'PREVIOUS_VERSION'`` (default)
+-   ``'2.6'`` (default)
 
     This implementation uses the same request fingerprinting algorithm as
-    Scrapy PREVIOUS_VERSION and earlier versions.
+    Scrapy 2.6 and earlier versions.
 
     Even though this is the default value for backward compatibility reasons,
     it is a deprecated value.
 
--   ``'VERSION'``
+-   ``'2.7'``
 
-    This implementation was introduced in Scrapy VERSION to fix an issue of the
+    This implementation was introduced in Scrapy 2.7 to fix an issue of the
     previous implementation.
 
     New projects should use this value. The :command:`startproject` command
     sets this value in the generated ``settings.py`` file.
 
-If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are
+If you are using the default value (``'2.6'``) for this setting, and you are
 using Scrapy components where changing the request fingerprinting algorithm
 would cause undesired results, you need to carefully decide when to change the
 value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
-setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request
+setting to a custom request fingerprinter class that implements the 2.6 request
 fingerprinting algorithm and does not log this warning (
-:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a
+:ref:`2.6-request-fingerprinter` includes an example implementation of such a
 class).
 
 Scenarios where changing the request fingerprinting algorithm may cause
@@ -449,14 +449,14 @@ undesired results include, for example, using the HTTP cache middleware (see
 Changing the request fingerprinting algorithm would invalidade the current
 cache, requiring you to redownload all requests again.
 
-Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in
+Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in
 your settings to switch already to the request fingerprinting implementation
 that will be the only request fingerprinting implementation available in a
 future version of Scrapy, and remove the deprecation warning triggered by using
-the default value (``'PREVIOUS_VERSION'``).
+the default value (``'2.6'``).
 
 
-.. _PREVIOUS_VERSION-request-fingerprinter:
+.. _2.6-request-fingerprinter:
 .. _custom-request-fingerprinter:
 
 Writing your own request fingerprinter
@@ -464,6 +464,8 @@ Writing your own request fingerprinter
 
 A request fingerprinter is a class that must implement the following method:
 
+.. currentmodule:: None
+
 .. method:: fingerprint(self, request)
 
    Return a :class:`bytes` object that uniquely identifies *request*.
@@ -476,6 +478,7 @@ A request fingerprinter is a class that must implement the following method:
 Additionally, it may also implement the following methods:
 
 .. classmethod:: from_crawler(cls, crawler)
+   :noindex:
 
    If present, this class method is called to create a request fingerprinter
    instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
@@ -495,11 +498,13 @@ Additionally, it may also implement the following methods:
    :class:`~scrapy.settings.Settings` object. It must return a new instance of
    the request fingerprinter.
 
-The ``fingerprint`` method of the default request fingerprinter,
+.. currentmodule:: scrapy.http
+
+The :meth:`fingerprint` method of the default request fingerprinter,
 :class:`scrapy.utils.request.RequestFingerprinter`, uses
 :func:`scrapy.utils.request.fingerprint` with its default parameters. For some
-common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well
-in your ``fingerprint`` method implementation:
+common use cases you can use :func:`scrapy.utils.request.fingerprint` as well
+in your :meth:`fingerprint` method implementation:
 
 .. autofunction:: scrapy.utils.request.fingerprint
 
@@ -519,7 +524,7 @@ account::
 
 You can also write your own fingerprinting logic from scratch.
 
-However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure
+However, if you do not use :func:`scrapy.utils.request.fingerprint`, make sure
 you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
 
 -   Caching saves CPU by ensuring that fingerprints are calculated only once
@@ -553,7 +558,7 @@ If you need to be able to override the request fingerprinting for arbitrary
 requests from your spider callbacks, you may implement a request fingerprinter
 that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
 when available, and then falls back to
-:func:`~scrapy.utils.request.fingerprint`. For example::
+:func:`scrapy.utils.request.fingerprint`. For example::
 
     from scrapy.utils.request import fingerprint
 
@@ -564,8 +569,8 @@ when available, and then falls back to
                 return request.meta['fingerprint']
             return fingerprint(request)
 
-If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION
-without using the deprecated ``'PREVIOUS_VERSION'`` value of the
+If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6
+without using the deprecated ``'2.6'`` value of the
 :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
 request fingerprinter::
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0b1ef71cf..40bcda288 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1642,7 +1642,7 @@ install the default reactor defined by Twisted for the current platform. This
 is to maintain backward compatibility and avoid possible problems caused by
 using a non-default reactor.
 
-.. versionchanged:: VERSION
+.. versionchanged:: 2.7
    The :command:`startproject` command now sets this setting to
    ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated
    ``settings.py`` file.
@@ -1661,14 +1661,14 @@ Scope: ``spidermiddlewares.urllength``
 
 The maximum URL length to allow for crawled URLs.
 
-This setting can act as a stopping condition in case of URLs of ever-increasing 
-length, which may be caused for example by a programming error either in the 
-target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and 
+This setting can act as a stopping condition in case of URLs of ever-increasing
+length, which may be caused for example by a programming error either in the
+target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
 :setting:`DEPTH_LIMIT`.
 
 Use ``0`` to allow URLs of any length.
 
-The default value is copied from the `Microsoft Internet Explorer maximum URL 
+The default value is copied from the `Microsoft Internet Explorer maximum URL
 length`_, even though this setting exists for different reasons.
 
 .. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 816cb5e03..303401a3c 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -105,17 +105,17 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         :class:`~scrapy.Request` objects and :ref:`item objects
         <topics-items>`.
 
-        .. versionchanged:: VERSION
+        .. versionchanged:: 2.7
            This method may be defined as an :term:`asynchronous generator`, in
            which case ``result`` is an :term:`asynchronous iterable`.
 
         Consider defining this method as an :term:`asynchronous generator`,
         which will be a requirement in a future version of Scrapy. However, if
         you plan on sharing your spider middleware with other people, consider
-        either :ref:`enforcing Scrapy VERSION <enforce-component-requirements>`
+        either :ref:`enforcing Scrapy 2.7 <enforce-component-requirements>`
         as a minimum requirement of your spider middleware, or :ref:`making
         your spider middleware universal <universal-spider-middleware>` so that
-        it works with Scrapy versions earlier than Scrapy VERSION.
+        it works with Scrapy versions earlier than Scrapy 2.7.
 
         :param response: the response which generated this output from the
           spider
@@ -130,7 +130,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
     .. method:: process_spider_output_async(response, result, spider)
 
-        .. versionadded:: VERSION
+        .. versionadded:: 2.7
 
         If defined, this method must be an :term:`asynchronous generator`,
         which will be called instead of :meth:`process_spider_output` if
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index ff86af125..29ff028be 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -248,7 +248,7 @@ REFERER_ENABLED = True
 REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
 
 REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
-REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION'
+REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.6'
 
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index c0c34e986..bbf60982c 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -88,5 +88,5 @@ ROBOTSTXT_OBEY = True
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
 
 # Set settings whose default value is deprecated to a future-proof value
-REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION'
+REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
 TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index cf33317ce..fbddc41fb 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -236,10 +236,10 @@ class RequestFingerprinter:
                 'REQUEST_FINGERPRINTER_IMPLEMENTATION'
             )
         else:
-            implementation = 'PREVIOUS_VERSION'
-        if implementation == 'PREVIOUS_VERSION':
+            implementation = '2.6'
+        if implementation == '2.6':
             message = (
-                '\'PREVIOUS_VERSION\' is a deprecated value for the '
+                '\'2.6\' is a deprecated value for the '
                 '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n'
                 '\n'
                 'It is also the default value. In other words, it is normal '
@@ -254,14 +254,14 @@ class RequestFingerprinter:
             )
             warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
             self._fingerprint = _request_fingerprint_as_bytes
-        elif implementation == 'VERSION':
+        elif implementation == '2.7':
             self._fingerprint = fingerprint
         else:
             raise ValueError(
                 f'Got an invalid value on setting '
                 f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '
-                f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) '
-                f'and \'VERSION\'.'
+                f'{implementation!r}. Valid values are \'2.6\' (deprecated) '
+                f'and \'2.7\'.'
             )
 
     def fingerprint(self, request):
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 0b828f7c0..445cd2e3a 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -65,7 +65,7 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
     # Set by default settings that prevent deprecation warnings.
     settings = {}
     if prevent_warnings:
-        settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION'
+        settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = '2.7'
     settings.update(settings_dict or {})
     runner = CrawlerRunner(settings)
     return runner.create_crawler(spidercls or Spider)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 8be4b6fe1..5383ec652 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -350,7 +350,7 @@ with multiples lines
 
     @defer.inlineCallbacks
     def test_crawl_multiple(self):
-        runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})
+        runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'})
         runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
         runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
 
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index c61d461f7..da6024c2b 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -110,7 +110,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
                 'LOG_LEVEL': 'INFO',
                 'LOG_FILE': log_file,
                 # settings to avoid extra warnings
-                'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
+                'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
                 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
             }
 
@@ -235,7 +235,7 @@ class NoRequestsSpider(scrapy.Spider):
 class CrawlerRunnerHasSpider(unittest.TestCase):
 
     def _runner(self):
-        return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})
+        return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'})
 
     @defer.inlineCallbacks
     def test_crawler_runner_bootstrap_successful(self):
@@ -283,14 +283,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         if self.reactor_pytest == 'asyncio':
             CrawlerRunner(settings={
                 "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-                "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",
+                "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
             })
         else:
             msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
             with self.assertRaisesRegex(Exception, msg):
                 runner = CrawlerRunner(settings={
                     "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-                    "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",
+                    "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
                 })
                 yield runner.crawl(NoRequestsSpider)
 
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 8a37a8ebe..6ebb716b0 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -51,7 +51,7 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_df_from_crawler_scheduler(self):
         settings = {'DUPEFILTER_DEBUG': True,
                     'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
-                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
+                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
@@ -60,7 +60,7 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_df_from_settings_scheduler(self):
         settings = {'DUPEFILTER_DEBUG': True,
                     'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter,
-                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
+                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
@@ -68,7 +68,7 @@ class RFPDupeFilterTest(unittest.TestCase):
 
     def test_df_direct_scheduler(self):
         settings = {'DUPEFILTER_CLASS': DirectDupeFilter,
-                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
+                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertEqual(scheduler.df.method, 'n/a')
@@ -172,7 +172,7 @@ class RFPDupeFilterTest(unittest.TestCase):
         with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': False,
                         'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
-                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
+                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             spider = SimpleSpider.from_crawler(crawler)
             dupefilter = _get_dupefilter(crawler=crawler)
@@ -199,7 +199,7 @@ class RFPDupeFilterTest(unittest.TestCase):
         with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': True,
                         'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
-                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
+                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             spider = SimpleSpider.from_crawler(crawler)
             dupefilter = _get_dupefilter(crawler=crawler)
@@ -233,7 +233,7 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_log_debug_default_dupefilter(self):
         with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': True,
-                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
+                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
             spider = SimpleSpider.from_crawler(crawler)
             dupefilter = _get_dupefilter(crawler=crawler)
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index e46532a1c..0e174cd34 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -64,7 +64,7 @@ class FileDownloadCrawlTestCase(TestCase):
         self.tmpmediastore = self.mktemp()
         os.mkdir(self.tmpmediastore)
         self.settings = {
-            'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
             'ITEM_PIPELINES': {self.pipeline_class: 1},
             self.store_setting_key: self.tmpmediastore,
         }
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index ac66056ba..50a7755c1 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -52,7 +52,7 @@ class MockCrawler(Crawler):
             SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
             JOBDIR=jobdir,
             DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
-            REQUEST_FINGERPRINTER_IMPLEMENTATION='VERSION',
+            REQUEST_FINGERPRINTER_IMPLEMENTATION='2.7',
         )
         super().__init__(Spider, settings)
         self.engine = MockEngine(downloader=MockDownloader())
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 3719c7c9f..7a590f96c 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -98,7 +98,7 @@ class SpiderLoaderTest(unittest.TestCase):
         module = 'tests.test_spiderloader.test_spiders.spider1'
         runner = CrawlerRunner({
             'SPIDER_MODULES': [module],
-            'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
         })
 
         self.assertRaisesRegex(KeyError, 'Spider not found',
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 8bc7922b6..a92d9a0ac 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -505,7 +505,7 @@ class RequestFingerprinterTestCase(unittest.TestCase):
 
     def test_deprecated_implementation(self):
         settings = {
-            'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION',
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.6',
         }
         with warnings.catch_warnings(record=True) as logged_warnings:
             crawler = get_crawler(settings_dict=settings)
@@ -518,7 +518,7 @@ class RequestFingerprinterTestCase(unittest.TestCase):
 
     def test_recommended_implementation(self):
         settings = {
-            'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
         }
         with warnings.catch_warnings(record=True) as logged_warnings:
             crawler = get_crawler(settings_dict=settings)

From 20b79a0f2e47800bf4648c7f890c8170fc8f5ede Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 17 Oct 2022 19:09:22 +0600
Subject: [PATCH 696/716] =?UTF-8?q?Bump=20version:=202.6.2=20=E2=86=92=202?=
 =?UTF-8?q?.7.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 docs/news.rst    | 2 +-
 scrapy/VERSION   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 2e2f7949a..f88071685 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.6.2
+current_version = 2.7.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/docs/news.rst b/docs/news.rst
index d8b9fcd1e..1ec183a1d 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5,7 +5,7 @@ Release notes
 
 .. _release-2.7.0:
 
-Scrapy 2.7.0 (to be released)
+Scrapy 2.7.0 (2022-10-17)
 -----------------------------
 
 Highlights:
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 097a15a2a..24ba9a38d 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.6.2
+2.7.0

From b33244e2f0d877b8911f949308222db0b076d665 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Fri, 21 Oct 2022 19:17:04 +0500
Subject: [PATCH 697/716] Fix the flake8 per-file ignore syntax (#5688)

---
 .flake8              | 17 +++++++++--------
 scrapy/utils/url.py  |  1 +
 tests/test_loader.py |  2 +-
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/.flake8 b/.flake8
index d7aebc24b..0c64d009e 100644
--- a/.flake8
+++ b/.flake8
@@ -6,16 +6,17 @@ ignore = W503
 exclude =
     docs/conf.py
 
+per-file-ignores =
 # Exclude files that are meant to provide top-level imports
 # E402: Module level import not at top of file
 # F401: Module imported but unused
-    scrapy/__init__.py E402
-    scrapy/core/downloader/handlers/http.py F401
-    scrapy/http/__init__.py F401
-    scrapy/linkextractors/__init__.py E402 F401
-    scrapy/selector/__init__.py F401
-    scrapy/spiders/__init__.py E402 F401
+    scrapy/__init__.py:E402
+    scrapy/core/downloader/handlers/http.py:F401
+    scrapy/http/__init__.py:F401
+    scrapy/linkextractors/__init__.py:E402,F401
+    scrapy/selector/__init__.py:F401
+    scrapy/spiders/__init__.py:E402,F401
 
     # Issues pending a review:
-    scrapy/utils/url.py F403 F405
-    tests/test_loader.py E741
+    scrapy/utils/url.py:F403,F405
+    tests/test_loader.py:E741
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 4d5e9ae82..21201ace5 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -34,6 +34,7 @@ def url_has_any_extension(url, extensions):
     lowercase_path = parse_url(url).path.lower()
     return any(lowercase_path.endswith(ext) for ext in extensions)
 
+
 def parse_url(url, encoding=None):
     """Return urlparsed url from the given argument (which could be an already
     parsed url)
diff --git a/tests/test_loader.py b/tests/test_loader.py
index c0937b349..b3e44d36b 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -295,7 +295,7 @@ class SelectortemLoaderTest(unittest.TestCase):
 
         l.add_css('name', 'div::text')
         self.assertEqual(l.get_output_value('name'), ['Marta'])
-    
+
     def test_init_method_with_base_response(self):
         """Selector should be None after initialization"""
         response = Response("https://scrapy.org")

From b61b71c6f015d45f6e98a4280bf4993517180045 Mon Sep 17 00:00:00 2001
From: Godson-Gnanaraj <lgodsongnanaraj@gmail.com>
Date: Tue, 25 Oct 2022 08:44:43 +0530
Subject: [PATCH 698/716] Replace indentation of source before parsing with
 ast.

closes #5323
---
 scrapy/utils/misc.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 1221b39b2..c0258c8d9 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -226,7 +226,14 @@ def is_generator_with_return_value(callable):
         return value is None or isinstance(value, ast.NameConstant) and value.value is None
 
     if inspect.isgeneratorfunction(callable):
-        code = re.sub(r"^[\t ]+", "", inspect.getsource(callable))
+        pattern = r"(^[\t ]+)"
+        src = inspect.getsource(callable)
+        match = re.match(pattern, src)  # Find indentation
+        code = re.sub(pattern, "", src)
+        if match:
+            # Remove indentation
+            code = re.sub(f"\n{match.group(0)}", "\n", code)
+
         tree = ast.parse(code)
         for node in walk_callable(tree):
             if isinstance(node, ast.Return) and not returns_none(node):

From f4e2a10ed6a44300738bd3701ea62b432c9c1a06 Mon Sep 17 00:00:00 2001
From: Kaushal Sharma <kaushalsharma880@gmail.com>
Date: Tue, 25 Oct 2022 02:45:46 -0700
Subject: [PATCH 699/716] =?UTF-8?q?Image.ANTIALIAS=20=E2=86=92=20Image.Res?=
 =?UTF-8?q?ampling.LANCZOS=20(#5692)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scrapy/pipelines/images.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 6b97190ee..67b3224b3 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -160,7 +160,14 @@ class ImagesPipeline(FilesPipeline):
 
         if size:
             image = image.copy()
-            image.thumbnail(size, self._Image.ANTIALIAS)
+            try:
+                # Image.Resampling.LANCZOS was added in Pillow 9.1.0
+                # remove this try except block,
+                # when updating the minimum requirements for Pillow.
+                resampling_filter = self._Image.Resampling.LANCZOS
+            except AttributeError:
+                resampling_filter = self._Image.ANTIALIAS
+            image.thumbnail(size, resampling_filter)
 
         buf = BytesIO()
         image.save(buf, 'JPEG')

From 830e1c5dd85618a27749bbe41d35b11fb2bdd348 Mon Sep 17 00:00:00 2001
From: Godson-Gnanaraj <lgodsongnanaraj@gmail.com>
Date: Wed, 26 Oct 2022 01:26:54 +0530
Subject: [PATCH 700/716] Add test for parsing decorated methods

---
 ...t_return_with_argument_inside_generator.py | 83 +++++++++++++++++++
 1 file changed, 83 insertions(+)

diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
index 1c85ca353..72277d701 100644
--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py
+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
@@ -165,6 +165,89 @@ https://example.org
             warn_on_generator_with_return_value(None, l2)
             self.assertEqual(len(w), 0)
 
+    def test_generators_return_none_with_decorator(self):
+        def decorator(func):
+            def inner_func():
+                func()
+            return inner_func
+
+        @decorator
+        def f3():
+            yield 1
+            return None
+
+        @decorator
+        def g3():
+            yield 1
+            return
+
+        @decorator
+        def h3():
+            yield 1
+
+        @decorator
+        def i3():
+            yield 1
+            yield from generator_that_returns_stuff()
+
+        @decorator
+        def j3():
+            yield 1
+
+            def helper():
+                return 0
+
+            yield helper()
+
+        @decorator
+        def k3():
+            """
+docstring
+            """
+            url = """
+https://example.org
+        """
+            yield url
+            return
+
+        @decorator
+        def l3():
+            return
+
+        assert not is_generator_with_return_value(top_level_return_none)
+        assert not is_generator_with_return_value(f3)
+        assert not is_generator_with_return_value(g3)
+        assert not is_generator_with_return_value(h3)
+        assert not is_generator_with_return_value(i3)
+        assert not is_generator_with_return_value(j3)  # not recursive
+        assert not is_generator_with_return_value(k3)  # not recursive
+        assert not is_generator_with_return_value(l3)
+
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, top_level_return_none)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, f3)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, g3)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, h3)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, i3)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, j3)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, k3)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, l3)
+            self.assertEqual(len(w), 0)
+
     @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error)
     def test_indentation_error(self):
         with warnings.catch_warnings(record=True) as w:

From b0ddffc47b9cee5e6146497b42de3787da76d2ad Mon Sep 17 00:00:00 2001
From: Godson-Gnanaraj <lgodsongnanaraj@gmail.com>
Date: Wed, 26 Oct 2022 06:53:43 +0530
Subject: [PATCH 701/716] Misc. changes:   - compile regex   - readability
 improvements

---
 scrapy/utils/misc.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index c0258c8d9..4d4fb9600 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -226,13 +226,13 @@ def is_generator_with_return_value(callable):
         return value is None or isinstance(value, ast.NameConstant) and value.value is None
 
     if inspect.isgeneratorfunction(callable):
-        pattern = r"(^[\t ]+)"
         src = inspect.getsource(callable)
-        match = re.match(pattern, src)  # Find indentation
-        code = re.sub(pattern, "", src)
+        pattern = re.compile(r"(^[\t ]+)")
+        code = pattern.sub("", src)
+
+        match = pattern.match(src)  # finds indentation
         if match:
-            # Remove indentation
-            code = re.sub(f"\n{match.group(0)}", "\n", code)
+            code = re.sub(f"\n{match.group(0)}", "\n", code)  # remove indentation
 
         tree = ast.parse(code)
         for node in walk_callable(tree):

From 2464939b7ee8381f904f7b0625aee6de0989b6c8 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Wed, 26 Oct 2022 15:58:20 -0300
Subject: [PATCH 702/716] Fixed deprecation warning in scrapy.core.engine
 (#5589)

* Change `download` function logic

* Fix CI error in 3.7 checks

* Make `spider` parameter optional in `_download` function, assign spider value from self if `None`
---
 scrapy/core/engine.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 6602f661d..1228e78da 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -257,9 +257,7 @@ class ExecutionEngine:
 
     def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
         """Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
-        if spider is None:
-            spider = self.spider
-        else:
+        if spider is not None:
             warnings.warn(
                 "Passing a 'spider' argument to ExecutionEngine.download is deprecated",
                 category=ScrapyDeprecationWarning,
@@ -267,7 +265,7 @@ class ExecutionEngine:
             )
             if spider is not self.spider:
                 logger.warning("The spider '%s' does not match the open spider", spider.name)
-        if spider is None:
+        if self.spider is None:
             raise RuntimeError(f"No open spider to crawl: {request}")
         return self._download(request, spider).addBoth(self._downloaded, request, spider)
 
@@ -278,11 +276,14 @@ class ExecutionEngine:
         self.slot.remove_request(request)
         return self.download(result, spider) if isinstance(result, Request) else result
 
-    def _download(self, request: Request, spider: Spider) -> Deferred:
+    def _download(self, request: Request, spider: Optional[Spider]) -> Deferred:
         assert self.slot is not None  # typing
 
         self.slot.add_request(request)
 
+        if spider is None:
+            spider = self.spider
+
         def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
             if not isinstance(result, (Response, Request)):
                 raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}")

From b394f2165acd662e55762d524b916f146291e422 Mon Sep 17 00:00:00 2001
From: Andrei Andrukhovich <aaandrukhovich@yandex.ru>
Date: Wed, 26 Oct 2022 23:11:28 +0300
Subject: [PATCH 703/716] =?UTF-8?q?Fix=20typo:=20[they]=20depends=20?=
 =?UTF-8?q?=E2=86=92=20depend=20(#5694)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/intro/install.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 9ab479edd..2c2079f68 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -52,7 +52,7 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
 * `twisted`_, an asynchronous networking framework
 * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
 
-Some of these packages themselves depends on non-Python packages
+Some of these packages themselves depend on non-Python packages
 that might require additional installation steps depending on your platform.
 Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.
 

From a214147359b8840abbb67bbe2a4a4066b271d2a5 Mon Sep 17 00:00:00 2001
From: Johanan Idicula <git@johanan.dev>
Date: Wed, 26 Oct 2022 21:43:35 -0400
Subject: [PATCH 704/716] ci: Update macos runner

The GitHub Actions macos-10.15 runner image is now deprecated, and GitHub
Actions has begun to temporarily fail jobs referencing it during brownout
periods. The image will be fully unsupported by 2022-12-01, which is just about
a month away.

This change updates the macOS runner image to the latest generally-available
version, to help reduce spurious CI failures during the brownout periods, and to
stay abreast of the sunsetting of the macos-10.15 image.

See also: actions/runner-images#5583
---
 .github/workflows/tests-macos.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index fdb9f4980..61f1857f8 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -3,7 +3,7 @@ on: [push, pull_request]
 
 jobs:
   tests:
-    runs-on: macos-10.15
+    runs-on: macos-11
     strategy:
       fail-fast: false
       matrix:

From ca50af645390e38299082d9ef4682c4be482ae70 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 27 Oct 2022 17:12:26 +0600
Subject: [PATCH 705/716] Remove an unused import.

---
 tests/test_pipeline_images.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index c4ce2736f..c189d08bf 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -4,7 +4,6 @@ import io
 import random
 from shutil import rmtree
 from tempfile import mkdtemp
-from unittest import skipIf
 from warnings import catch_warnings
 
 import attr

From b6541830849a6edd41da6b539d272790a4661d7d Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Thu, 27 Oct 2022 17:00:36 +0500
Subject: [PATCH 706/716] Use Python 3.11 as the default in CI (#5696)

---
 .github/workflows/checks.yml        | 8 ++++----
 .github/workflows/publish.yml       | 2 +-
 .github/workflows/tests-macos.yml   | 2 +-
 .github/workflows/tests-ubuntu.yml  | 9 +++------
 .github/workflows/tests-windows.yml | 7 +++++++
 .readthedocs.yml                    | 2 +-
 6 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 439dfee51..8c1ae4bd3 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -8,10 +8,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: "3.10"
+        - python-version: "3.11"
           env:
             TOXENV: security
-        - python-version: "3.10"
+        - python-version: "3.11"
           env:
             TOXENV: flake8
         # Pylint requires installing reppy, which does not support Python 3.9
@@ -22,10 +22,10 @@ jobs:
         - python-version: 3.7
           env:
             TOXENV: typing
-        - python-version: "3.10"  # Keep in sync with .readthedocs.yml
+        - python-version: "3.11"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs
-        - python-version: "3.10"
+        - python-version: "3.11"
           env:
             TOXENV: twinecheck
 
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index f6b098b80..991b0b6e8 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -12,7 +12,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.11"
 
     - name: Check Tag
       id: check-release-tag
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 61f1857f8..174d245ca 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index d2bfe4a5f..9c3ce8115 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -17,13 +17,10 @@ jobs:
         - python-version: "3.10"
           env:
             TOXENV: py
-        - python-version: "3.10"
-          env:
-            TOXENV: asyncio
-        - python-version: "3.11.0-rc.2"
+        - python-version: "3.11"
           env:
             TOXENV: py
-        - python-version: "3.11.0-rc.2"
+        - python-version: "3.11"
           env:
             TOXENV: asyncio
         - python-version: pypy3.9
@@ -57,7 +54,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install system libraries
-      if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2'
+      if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned')
       run: |
         sudo apt-get update
         sudo apt-get install libxml2-dev libxslt-dev
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 14683fd53..f60c48841 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -23,6 +23,13 @@ jobs:
         - python-version: "3.10"
           env:
             TOXENV: asyncio
+# no binary package for lxml for 3.11 yet
+#        - python-version: "3.11"
+#          env:
+#            TOXENV: py
+#        - python-version: "3.11"
+#          env:
+#            TOXENV: asyncio
 
     steps:
     - uses: actions/checkout@v3
diff --git a/.readthedocs.yml b/.readthedocs.yml
index 390be3749..e71d34f3a 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -9,7 +9,7 @@ build:
   tools:
     # For available versions, see:
     # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
-    python: "3.10"  # Keep in sync with .github/workflows/checks.yml
+    python: "3.11"  # Keep in sync with .github/workflows/checks.yml
 
 python:
   install:

From 3a34fa839938d7640e56f3ce12ba60aed31a6ecf Mon Sep 17 00:00:00 2001
From: Godson <30664729+Godson-Gnanaraj@users.noreply.github.com>
Date: Thu, 27 Oct 2022 17:32:12 +0530
Subject: [PATCH 707/716] Get the event loop from event_loop_policy to avoid a
 deprecation warning (#5689)

---
 scrapy/utils/defer.py       |  8 +++++---
 scrapy/utils/reactor.py     | 21 ++++++++++++++++-----
 tests/test_utils_asyncio.py |  6 ++++--
 3 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 8fcf31cab..38aefd6d0 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -26,7 +26,7 @@ from twisted.python import failure
 from twisted.python.failure import Failure
 
 from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.reactor import is_asyncio_reactor_installed
+from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy
 
 
 def defer_fail(_failure: Failure) -> Deferred:
@@ -269,7 +269,8 @@ def deferred_from_coro(o) -> Any:
             return ensureDeferred(o)
         else:
             # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
-            return Deferred.fromFuture(asyncio.ensure_future(o))
+            event_loop = get_asyncio_event_loop_policy().get_event_loop()
+            return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))
     return o
 
 
@@ -320,7 +321,8 @@ def deferred_to_future(d: Deferred) -> Future:
                 d = treq.get('https://example.com/additional')
                 additional_response = await deferred_to_future(d)
     """
-    return d.asFuture(asyncio.get_event_loop())
+    policy = get_asyncio_event_loop_policy()
+    return d.asFuture(policy.get_event_loop())
 
 
 def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 652733ce8..ddf354d88 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -51,6 +51,19 @@ class CallLaterOnce:
         return self._func(*self._a, **self._kw)
 
 
+def get_asyncio_event_loop_policy():
+    policy = asyncio.get_event_loop_policy()
+    if (
+        sys.version_info >= (3, 8)
+        and sys.platform == "win32"
+        and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy)
+    ):
+        policy = asyncio.WindowsSelectorEventLoopPolicy()
+        asyncio.set_event_loop_policy(policy)
+
+    return policy
+
+
 def install_reactor(reactor_path, event_loop_path=None):
     """Installs the :mod:`~twisted.internet.reactor` with the specified
     import path. Also installs the asyncio event loop with the specified import
@@ -58,16 +71,14 @@ def install_reactor(reactor_path, event_loop_path=None):
     reactor_class = load_object(reactor_path)
     if reactor_class is asyncioreactor.AsyncioSelectorReactor:
         with suppress(error.ReactorAlreadyInstalledError):
-            if sys.version_info >= (3, 8) and sys.platform == "win32":
-                policy = asyncio.get_event_loop_policy()
-                if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy):
-                    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+            policy = get_asyncio_event_loop_policy()
             if event_loop_path is not None:
                 event_loop_class = load_object(event_loop_path)
                 event_loop = event_loop_class()
                 asyncio.set_event_loop(event_loop)
             else:
-                event_loop = asyncio.get_event_loop()
+                event_loop = policy.get_event_loop()
+
             asyncioreactor.install(eventloop=event_loop)
     else:
         *module, _ = reactor_path.split(".")
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 295323e4d..741c6a505 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -1,3 +1,4 @@
+import warnings
 from unittest import TestCase
 
 from pytest import mark
@@ -13,5 +14,6 @@ class AsyncioTest(TestCase):
         self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
 
     def test_install_asyncio_reactor(self):
-        # this should do nothing
-        install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+        with warnings.catch_warnings(record=True) as w:
+            install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+            self.assertEqual(len(w), 0)

From b71d0292d5ff85d652e2943ee1a727a128b55594 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 27 Oct 2022 18:13:47 +0600
Subject: [PATCH 708/716] Add a test for processing partial callbacks.

---
 .../test_return_with_argument_inside_generator.py         | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
index 72277d701..562f72fee 100644
--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py
+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
@@ -1,5 +1,6 @@
 import unittest
 import warnings
+from functools import partial
 from unittest import mock
 
 from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value
@@ -254,3 +255,10 @@ https://example.org
             warn_on_generator_with_return_value(None, top_level_return_none)
             self.assertEqual(len(w), 1)
             self.assertIn('Unable to determine', str(w[0].message))
+
+    def test_partial(self):
+        def cb(arg1, arg2):
+            yield {}
+
+        partial_cb = partial(cb, arg1=42)
+        assert not is_generator_with_return_value(partial_cb)

From fd692f309105d917f5f46bd00a88c550d6cc7da3 Mon Sep 17 00:00:00 2001
From: Magnus Offermanns <maoffermanns@edu.aau.at>
Date: Thu, 27 Oct 2022 14:43:31 +0200
Subject: [PATCH 709/716] Prevent running the -O and -t command-line options
 together (#5605)

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 docs/topics/commands.rst    | 20 ++++++++++++++++++++
 scrapy/commands/__init__.py |  6 ++++--
 scrapy/utils/conf.py        | 19 ++++++++++++++++---
 3 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 8c0b8e55f..362190116 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -271,11 +271,31 @@ crawl
 
 Start crawling using a spider.
 
+Supported options:
+
+* ``-h, --help``: show a help message and exit
+
+* ``-a NAME=VALUE``: set a spider argument (may be repeated)
+
+* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``)
+
+* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)
+
+* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O``
+
 Usage examples::
 
     $ scrapy crawl myspider
     [ ... myspider starts crawling ... ]
 
+    $ scrapy -o myfile:csv myspider
+    [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
+
+    $ scrapy -O myfile:json myspider
+    [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ]
+
+    $ scrapy -o myfile -t csv myspider
+    [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ]
 
 .. command:: check
 
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index fb304b8c0..49c4e2f42 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
         parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                             help="set spider argument (may be repeated)")
         parser.add_argument("-o", "--output", metavar="FILE", action="append",
-                            help="append scraped items to the end of FILE (use - for stdout)")
+                            help="append scraped items to the end of FILE (use - for stdout),"
+                                 " to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)")
         parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
-                            help="dump scraped items into FILE, overwriting any existing file")
+                            help="dump scraped items into FILE, overwriting any existing file,"
+                                 " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)")
         parser.add_argument("-t", "--output-format", metavar="FORMAT",
                             help="format to use for dumping items")
 
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 00cc53725..6404edda6 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -155,6 +155,15 @@ def feed_process_params_from_cli(settings, output, output_format=None,
             raise UsageError(
                 "Please use only one of -o/--output and -O/--overwrite-output"
             )
+        if output_format:
+            raise UsageError(
+                "-t/--output-format is a deprecated command line option"
+                " and does not work in combination with -O/--overwrite-output."
+                " To specify a format please specify it after a colon at the end of the"
+                " output URI (i.e. -O <URI>:<FORMAT>)."
+                " Example working in the tutorial: "
+                "scrapy crawl quotes -O quotes.json:json"
+            )
         output = overwrite_output
         overwrite = True
 
@@ -162,9 +171,13 @@ def feed_process_params_from_cli(settings, output, output_format=None,
         if len(output) == 1:
             check_valid_format(output_format)
             message = (
-                'The -t command line option is deprecated in favor of '
-                'specifying the output format within the output URI. See the '
-                'documentation of the -o and -O options for more information.'
+                "The -t/--output-format command line option is deprecated in favor of "
+                "specifying the output format within the output URI using the -o/--output or the"
+                " -O/--overwrite-output option (i.e. -o/-O <URI>:<FORMAT>). See the documentation"
+                " of the -o or -O option or the following examples for more information. "
+                "Examples working in the tutorial: "
+                "scrapy crawl quotes -o quotes.csv:csv   or   "
+                "scrapy crawl quotes -O quotes.json:json"
             )
             warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
             return {output[0]: {'format': output_format}}

From bd9e482c2f0db92065708c8291be6e8bc1f05218 Mon Sep 17 00:00:00 2001
From: iamkaushal <kaushalsharma880@gmail.com>
Date: Thu, 27 Oct 2022 23:21:55 +0530
Subject: [PATCH 710/716] added typing.io and typing.re in pytest warning
 filter to ignore

---
 pytest.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pytest.ini b/pytest.ini
index af0f2fb6e..f5fbf2529 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -24,3 +24,5 @@ markers =
 filterwarnings =
     ignore:scrapy.downloadermiddlewares.decompression is deprecated
     ignore:Module scrapy.utils.reqser is deprecated
+    ignore:typing.re is deprecated
+    ignore:typing.io is deprecated

From 9f45be439de8a3b9a6d201c33e98b408a73c02bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E0=B4=8E=E0=B4=A4=E0=B4=BF=E0=B4=B0=E0=B4=BE=E0=B4=B3?=
 =?UTF-8?q?=E0=B4=BF=E0=B4=95=E0=B5=8D=E0=B4=95=E0=B5=8A=E0=B4=B0=E0=B5=81?=
 =?UTF-8?q?=20=E0=B4=AA=E0=B5=8B=E0=B4=B0=E0=B4=BE=E0=B4=B3=E0=B4=BF?=
 <108031802+pankali@users.noreply.github.com>
Date: Fri, 28 Oct 2022 02:13:37 +0200
Subject: [PATCH 711/716] Update Code of Conduct to Contributor Covenant v2.1

---
 CODE_OF_CONDUCT.md | 152 +++++++++++++++++++++++++++++++--------------
 1 file changed, 104 insertions(+), 48 deletions(-)

diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index 902cd523e..3c8e4d1b5 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -1,77 +1,133 @@
+
 # Contributor Covenant Code of Conduct
 
 ## Our Pledge
 
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to make participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, gender identity and expression, level of experience,
-nationality, personal appearance, race, religion, or sexual identity and
-orientation.
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
 
 ## Our Standards
 
-Examples of behavior that contributes to creating a positive environment
-include:
+Examples of behavior that contributes to a positive environment for our
+community include:
 
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
 
-Examples of unacceptable behavior by participants include:
+Examples of unacceptable behavior include:
 
-* The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
 * Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
 * Other conduct which could reasonably be considered inappropriate in a
   professional setting
 
-## Our Responsibilities
+## Enforcement Responsibilities
 
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
 
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
 
 ## Scope
 
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
 
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at opensource@zyte.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
+reported to the community leaders responsible for enforcement at
+opensource@zyte.com.
+All complaints will be reviewed and investigated promptly and fairly.
 
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
 
 ## Attribution
 
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version].
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
 
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
 
-For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

From 3259a4252566f4130f14ce2acd03531a735dae90 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 30 Oct 2022 13:17:37 -0300
Subject: [PATCH 712/716] CrawlSpider: pass cb_kwargs from process_request

---
 scrapy/spiders/crawl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index edac082d0..2d9328633 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -102,9 +102,9 @@ class CrawlSpider(Spider):
                 request = self._build_request(rule_index, link)
                 yield rule.process_request(request, response)
 
-    def _callback(self, response):
+    def _callback(self, response, **cb_kwargs):
         rule = self._rules[response.meta['rule']]
-        return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow)
+        return self._parse_response(response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow)
 
     def _errback(self, failure):
         rule = self._rules[failure.request.meta['rule']]

From b18560315bda057610ecda165694f9c0f7445c1f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 30 Oct 2022 18:28:16 -0300
Subject: [PATCH 713/716] Add tests

---
 tests/spiders.py    | 11 +++++++++++
 tests/test_crawl.py | 11 +++++++++++
 2 files changed, 22 insertions(+)

diff --git a/tests/spiders.py b/tests/spiders.py
index 5ea8a4a21..7952e3d47 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -419,6 +419,17 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
         self.logger.info('[errback] status %i', failure.value.response.status)
 
 
+class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod):
+    name = 'crawl_spider_with_process_request_cb_kwargs'
+    rules = (
+        Rule(LinkExtractor(), callback='parse', follow=True, process_request="process_request"),
+    )
+
+    def process_request(self, request, response):
+        request.cb_kwargs["foo"] = "process_request"
+        return request
+
+
 class BytesReceivedCallbackSpider(MetaSpider):
 
     full_response_length = 2**18
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 5383ec652..5ec96e4a7 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -41,6 +41,7 @@ from tests.spiders import (
     CrawlSpiderWithAsyncGeneratorCallback,
     CrawlSpiderWithErrback,
     CrawlSpiderWithParseMethod,
+    CrawlSpiderWithProcessRequestCallbackKeywordArguments,
     DelaySpider,
     DuplicateStartRequestsSpider,
     FollowAllSpider,
@@ -426,6 +427,16 @@ class CrawlSpiderTestCase(TestCase):
         self.assertIn("[errback] status 500", str(log))
         self.assertIn("[errback] status 501", str(log))
 
+    @defer.inlineCallbacks
+    def test_crawlspider_process_request_cb_kwargs(self):
+        crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
+
+        self.assertIn("[parse] status 200 (foo: process_request)", str(log))
+        self.assertIn("[parse] status 201 (foo: process_request)", str(log))
+        self.assertIn("[parse] status 202 (foo: bar)", str(log))
+
     @defer.inlineCallbacks
     def test_async_def_parse(self):
         crawler = get_crawler(AsyncDefSpider)

From 940a73863bf7dcb16b3f2d9f5efb83efe4599712 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 1 Nov 2022 19:00:33 +0600
Subject: [PATCH 714/716] Release notes for 2.7.1.

---
 docs/news.rst | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/docs/news.rst b/docs/news.rst
index 1ec183a1d..b7c8c85b5 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,57 @@
 Release notes
 =============
 
+.. _release-2.7.1:
+
+Scrapy 2.7.1 (Not relased yet)
+------------------------------
+
+New features
+~~~~~~~~~~~~
+
+-   Relaxed the restriction introduced in 2.6.2 so that the
+    ``Proxy-Authentication`` header can again be set explicitly, as long as the
+    proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and
+    for as long as that proxy URL remains the same (:issue:`5626`).
+
+Bug fixes
+~~~~~~~~~
+
+-   Using ``-O``/``--overwrite-output`` and ``-t``/``--output-format`` options
+    together now produces an error instead of ignoring the former option
+    (:issue:`5516`, :issue:`5605`).
+
+-   Replaced deprecated :mod:`asyncio` APIs that implicitly use the current
+    event loop with code that explicitly requests a loop from the event loop
+    policy (:issue:`5685`, :issue:`5689`).
+
+-   Fixed uses of deprecated Scrapy APIs in the Scrapy itself (:issue:`5588`,
+    :issue:`5589`).
+
+-   Fixed uses of a deprecated Pillow API (:issue:`5684`, :issue:`5692`).
+
+-   Improved code that checks if generators return values, so that it no longer
+    fails on decorated methods and partial methods (:issue:`5323`,
+    :issue:`5592`, :issue:`5599`, :issue:`5691`).
+
+Documentation
+~~~~~~~~~~~~~
+
+-   Upgraded the Code of Conduct to Contributor Covenant v2.1 (:issue:`5698`).
+
+-   Fixed typos (:issue:`5681`, :issue:`5694`).
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+-   Re-enabled some erroneously disabled flake8 checks (:issue:`5688`).
+
+-   Ignored harmless deprecation warnings from :mod:`typing` in tests
+    (:issue:`5686`, :issue:`5697`).
+
+-   Modernized CI configuration (:issue:`5695`, :issue:`5696`).
+
+
 .. _release-2.7.0:
 
 Scrapy 2.7.0 (2022-10-17)

From 5ec175b8bb08f93c431d7d64d2389b90ec7a1f37 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 2 Nov 2022 13:54:00 +0600
Subject: [PATCH 715/716] Small relnotes fixes.

---
 docs/news.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index b7c8c85b5..257626526 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -14,7 +14,8 @@ New features
 -   Relaxed the restriction introduced in 2.6.2 so that the
     ``Proxy-Authentication`` header can again be set explicitly, as long as the
     proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and
-    for as long as that proxy URL remains the same (:issue:`5626`).
+    for as long as that proxy URL remains the same; this restores compatibility
+    with scrapy-zyte-smartproxy 2.1.0 and older (:issue:`5626`).
 
 Bug fixes
 ~~~~~~~~~
@@ -27,7 +28,7 @@ Bug fixes
     event loop with code that explicitly requests a loop from the event loop
     policy (:issue:`5685`, :issue:`5689`).
 
--   Fixed uses of deprecated Scrapy APIs in the Scrapy itself (:issue:`5588`,
+-   Fixed uses of deprecated Scrapy APIs in Scrapy itself (:issue:`5588`,
     :issue:`5589`).
 
 -   Fixed uses of a deprecated Pillow API (:issue:`5684`, :issue:`5692`).
@@ -51,7 +52,7 @@ Quality assurance
 -   Ignored harmless deprecation warnings from :mod:`typing` in tests
     (:issue:`5686`, :issue:`5697`).
 
--   Modernized CI configuration (:issue:`5695`, :issue:`5696`).
+-   Modernized our CI configuration (:issue:`5695`, :issue:`5696`).
 
 
 .. _release-2.7.0:

From 6ded3cf4cd134b615239babe28bb28c3ff524b05 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 2 Nov 2022 17:00:47 +0600
Subject: [PATCH 716/716] =?UTF-8?q?Bump=20version:=202.7.0=20=E2=86=92=202?=
 =?UTF-8?q?.7.1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bumpversion.cfg | 2 +-
 docs/news.rst    | 4 ++--
 scrapy/VERSION   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index f88071685..b949d81c4 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.7.0
+current_version = 2.7.1
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/docs/news.rst b/docs/news.rst
index 257626526..e5fc2971a 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5,8 +5,8 @@ Release notes
 
 .. _release-2.7.1:
 
-Scrapy 2.7.1 (Not relased yet)
-------------------------------
+Scrapy 2.7.1 (2022-11-02)
+-------------------------
 
 New features
 ~~~~~~~~~~~~
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 24ba9a38d..860487ca1 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.7.0
+2.7.1