From 2a540206a74af8d38a01aaa5a37adc1008cad6ca Mon Sep 17 00:00:00 2001 From: nramirezuy Date: Tue, 19 Aug 2014 13:57:00 -0300 Subject: [PATCH 001/568] fix xmliter namespace on selected node --- scrapy/utils/iterators.py | 22 +++++++++++++++------- tests/test_utils_iterators.py | 6 +++++- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 150b077ae..11b873f2e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -20,19 +20,27 @@ def xmliter(obj, nodename): - a unicode string - a string encoded as utf-8 """ - HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S) + DOCUMENT_HEADER_RE = re.compile(r'<\?xml[^>]+>\s*', re.S) HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S) + END_TAG_RE = re.compile(r'<\s*/([^\s>]+)\s*>', re.S) + NAMESPACE_RE = re.compile(r'((xmlns[:A-Za-z]*)=[^>\s]+)', re.S) text = _body_or_str(obj) - header_start = re.search(HEADER_START_RE, text) - header_start = header_start.group(1).strip() if header_start else '' - header_end = re_rsearch(HEADER_END_RE, text) - header_end = text[header_end[1]:].strip() if header_end else '' + document_header = re.search(DOCUMENT_HEADER_RE, text) + document_header = document_header.group().strip() if document_header else '' + header_end_idx = re_rsearch(HEADER_END_RE, text) + header_end = text[header_end_idx[1]:].strip() if header_end_idx else '' + namespaces = {} + if header_end: + for tagname in reversed(re.findall(END_TAG_RE, header_end)): + tag = re.search(r'<\s*%s.*?xmlns[:=][^>]*>' % tagname, text[:header_end_idx[1]], re.S) + if tag: + namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group())) r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) for match in r.finditer(text): - nodetext = header_start + match.group() + header_end - yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] + nodetext = document_header + match.group().replace(nodename, '%s %s' % (nodename, ' '.join(namespaces.values())), 1) + header_end + yield Selector(text=nodetext, type='xml') def csviter(obj, delimiter=None, headers=None, encoding=None): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index fe53f831f..8b5941605 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -61,7 +61,6 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url='http://mydummycompany.com', body=body) my_iter = self.xmliter(response, 'item') - node = next(my_iter) node.register_namespace('g', 'http://base.google.com/ns/1.0') self.assertEqual(node.xpath('title/text()').extract(), ['Item 1']) @@ -74,6 +73,11 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('id/text()').extract(), []) self.assertEqual(node.xpath('price/text()').extract(), []) + my_iter = self.xmliter(response, 'g:image_link') + node = next(my_iter) + node.register_namespace('g', 'http://base.google.com/ns/1.0') + self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + def test_xmliter_exception(self): body = u"""onetwo""" From ac111088c6470d55e4d698247a76a4938f4a6dc6 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 20:12:29 +0530 Subject: [PATCH 002/568] duplicate keys handled --- scrapy/http/request/form.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index c2413b431..660f4fc96 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ import six -from six.moves.urllib.parse import urljoin, urlencode +from six.moves.urllib.parse import urljoin, urlencode, urlsplit import lxml.html from parsel.selector import create_root_node @@ -33,7 +33,30 @@ class FormRequest(Request): self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') self._set_body(querystr) else: - self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr) + if urlsplit(self.url).query: + queries = (urlsplit(self.url).query + '&' + querystr).split('&') + else: + queries = querystr.split('&') + query_dict = {} + for i in range(len(queries)): + query_list = queries[i].split('=') + query_dict[query_list[0]] = query_list[1] + querystr = '' + query_key = list(query_dict.keys()) + for i in range(len(query_dict)): + querystr += (query_key[i] + '=' + query_dict[query_key[i]]) + if i!=len(query_dict)-1: + querystr += '&' + if urlsplit(self.url).fragment: + if urlsplit(self.url).query: + self._set_url(self.url[:self.url.index('?')+1]+querystr+'#'+urlsplit(self.url).fragment) + else: + self._set_url(self.url+'?'+querystr+'#'+urlsplit(self.url).fragment) + else: + if urlsplit(self.url).query: + self._set_url(self.url[:self.url.index('?')+1]+querystr) + else: + self._set_url(self.url+'?'+querystr) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From b5e454809e240839e76f050b4587abcd80c33765 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 20:12:31 +0530 Subject: [PATCH 003/568] Included test --- tests/test_http_request.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 58326a384..5a7cf3918 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -273,6 +273,19 @@ class FormRequestTest(RequestTest): def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) self.assertEqual(r1.body, b'') + + def test_formdata_overrides_querystring_duplicates(self): + data1 = {'a' : '1', 'b' : '2'} + data2 = [('a','one'), ('a','two'), ('c','three')] + + fs1 = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1)) + self.assertEqual(fs1[b'a'], [b'1']) + self.assertEqual(fs1[b'b'], [b'2']) + + fs2 = _qs(self.request_class('http://www.example.com?a=1&b=2&c=3', method='GET', formdata=data2)) + self.assertEqual(fs2[b'a'], [b'two']) + self.assertEqual(fs2[b'b'], [b'2']) + self.assertEqual(fs2[b'c'], [b'three']) def test_default_encoding_bytes(self): # using default encoding (utf-8) From 1bea5d307628a11d4010cdabb87b2a29a6c3c772 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 22:35:16 +0530 Subject: [PATCH 004/568] Fixed error --- scrapy/http/request/form.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 660f4fc96..f08d1f626 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -34,29 +34,32 @@ class FormRequest(Request): self._set_body(querystr) else: if urlsplit(self.url).query: - queries = (urlsplit(self.url).query + '&' + querystr).split('&') + queries = (urlsplit(self.url).query).split('&') else: queries = querystr.split('&') query_dict = {} + duplicate_key_passed=[] + for i in range(len(querystr.split('&'))): + query_list = querystr.split('&')[i].split('=') + duplicate_key_passed.append(query_list[0]) for i in range(len(queries)): query_list = queries[i].split('=') - query_dict[query_list[0]] = query_list[1] - querystr = '' + if duplicate_key_passed.count(query_list[0])<=1: + query_dict[query_list[0]] = query_list[1] + query_str = '' query_key = list(query_dict.keys()) for i in range(len(query_dict)): - querystr += (query_key[i] + '=' + query_dict[query_key[i]]) - if i!=len(query_dict)-1: - querystr += '&' + query_str += (query_key[i] + '=' + query_dict[query_key[i]] + '&') if urlsplit(self.url).fragment: if urlsplit(self.url).query: - self._set_url(self.url[:self.url.index('?')+1]+querystr+'#'+urlsplit(self.url).fragment) + self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + '#' + urlsplit(self.url).fragment) else: - self._set_url(self.url+'?'+querystr+'#'+urlsplit(self.url).fragment) + self._set_url(self.url + '?' + querystr + '#' + urlsplit(self.url).fragment) else: if urlsplit(self.url).query: - self._set_url(self.url[:self.url.index('?')+1]+querystr) + self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr) else: - self._set_url(self.url+'?'+querystr) + self._set_url(self.url + '?' + querystr) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 9a4bbd6d029f61dc60594fb8bca631c3a494f855 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 23:05:58 +0530 Subject: [PATCH 005/568] Update form.py --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index f08d1f626..1b487cbd0 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -44,7 +44,7 @@ class FormRequest(Request): duplicate_key_passed.append(query_list[0]) for i in range(len(queries)): query_list = queries[i].split('=') - if duplicate_key_passed.count(query_list[0])<=1: + if duplicate_key_passed.count(query_list[0])==0: query_dict[query_list[0]] = query_list[1] query_str = '' query_key = list(query_dict.keys()) From 4abcdcb306d8782e6b792ad36e40391a0bcbd0f3 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 23:22:53 +0530 Subject: [PATCH 006/568] Update test_http_request.py --- tests/test_http_request.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5a7cf3918..ec247a02d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -276,17 +276,10 @@ class FormRequestTest(RequestTest): def test_formdata_overrides_querystring_duplicates(self): data1 = {'a' : '1', 'b' : '2'} - data2 = [('a','one'), ('a','two'), ('c','three')] - - fs1 = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1)) + fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1)) self.assertEqual(fs1[b'a'], [b'1']) self.assertEqual(fs1[b'b'], [b'2']) - fs2 = _qs(self.request_class('http://www.example.com?a=1&b=2&c=3', method='GET', formdata=data2)) - self.assertEqual(fs2[b'a'], [b'two']) - self.assertEqual(fs2[b'b'], [b'2']) - self.assertEqual(fs2[b'c'], [b'three']) - def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b'one': b'two', b'price': b'\xc2\xa3 100'} From 023290dabc55c92ea2143a98f7753abaf21f74c9 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 23:50:31 +0530 Subject: [PATCH 007/568] Update test_http_request.py --- tests/test_http_request.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index ec247a02d..1ef84f1d6 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -275,10 +275,10 @@ class FormRequestTest(RequestTest): self.assertEqual(r1.body, b'') def test_formdata_overrides_querystring_duplicates(self): - data1 = {'a' : '1', 'b' : '2'} - fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1)) - self.assertEqual(fs1[b'a'], [b'1']) - self.assertEqual(fs1[b'b'], [b'2']) + data = {'a' : '1', 'b' : '2'} + fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) def test_default_encoding_bytes(self): # using default encoding (utf-8) From 9f1f4df9666f38f81adda5a41f3cac646a4f18dc Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Wed, 16 Jan 2019 22:59:41 +0530 Subject: [PATCH 008/568] Update test_http_request.py --- tests/test_http_request.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 1ef84f1d6..6c4cb10a8 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -275,8 +275,22 @@ class FormRequestTest(RequestTest): self.assertEqual(r1.body, b'') def test_formdata_overrides_querystring_duplicates(self): + #Without fragment data = {'a' : '1', 'b' : '2'} - fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data)) + fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + + #With fragment + data = (('a', '1'), ('b', '2')) + url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0] + fs = _qs(self.request_class(url, method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + + #Witout query + data = {'a' : '1', 'b' : '2'} + fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) From 3e67fa8fc1eaa2189cec08ce4000b3efc254ee1b Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Wed, 16 Jan 2019 23:01:38 +0530 Subject: [PATCH 009/568] Improved for better user readability --- scrapy/http/request/form.py | 45 ++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 1b487cbd0..fa5110382 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -33,33 +33,26 @@ class FormRequest(Request): self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') self._set_body(querystr) else: - if urlsplit(self.url).query: - queries = (urlsplit(self.url).query).split('&') + url_split = urlsplit(self.url) + formdata_key_list = [] + for k in querystr.split('&'): + formdata_key_list.append(k.split('=')[0]) + + if url_split.query: + queries = (url_split.query).split('&') + query_dict = {} + + for x in queries: + query = x.split('=') + if formdata_key_list.count(query[0])==0: + query_dict[query[0]] = query[1] + query_str = '' + for k, v in query_dict: + query_str += (k + '=' + v + '&') + + self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else '')) else: - queries = querystr.split('&') - query_dict = {} - duplicate_key_passed=[] - for i in range(len(querystr.split('&'))): - query_list = querystr.split('&')[i].split('=') - duplicate_key_passed.append(query_list[0]) - for i in range(len(queries)): - query_list = queries[i].split('=') - if duplicate_key_passed.count(query_list[0])==0: - query_dict[query_list[0]] = query_list[1] - query_str = '' - query_key = list(query_dict.keys()) - for i in range(len(query_dict)): - query_str += (query_key[i] + '=' + query_dict[query_key[i]] + '&') - if urlsplit(self.url).fragment: - if urlsplit(self.url).query: - self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + '#' + urlsplit(self.url).fragment) - else: - self._set_url(self.url + '?' + querystr + '#' + urlsplit(self.url).fragment) - else: - if urlsplit(self.url).query: - self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr) - else: - self._set_url(self.url + '?' + querystr) + self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else '')) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 6f86c93f366509ba05faf3b2cdc12f5e3ef6693a Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Wed, 16 Jan 2019 23:54:35 +0530 Subject: [PATCH 010/568] Increased test cases --- tests/test_http_request.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6c4cb10a8..571df0176 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -275,20 +275,27 @@ class FormRequestTest(RequestTest): self.assertEqual(r1.body, b'') def test_formdata_overrides_querystring_duplicates(self): - #Without fragment - data = {'a' : '1', 'b' : '2'} - fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) - self.assertEqual(fs[b'b'], [b'2']) - - #With fragment + #Both fragment and query in url data = (('a', '1'), ('b', '2')) url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) - #Witout query + #only query in url + data = {'a' : '1', 'b' : '2'} + fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + + #only fragment in url + data = (('a', '1'), ('b', '2')) + url = self.request_class('http://www.example.com/#fragment', method='GET', formdata=data).url.split('#')[0] + fs = _qs(self.request_class(url, method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + + #None of both in url data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) From 6be73f06c33a27cb3291a1e46d80574b4d23b10b Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Thu, 17 Jan 2019 23:50:58 +0530 Subject: [PATCH 011/568] Updated tests --- tests/test_http_request.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 571df0176..a17acf93b 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -277,23 +277,11 @@ class FormRequestTest(RequestTest): def test_formdata_overrides_querystring_duplicates(self): #Both fragment and query in url data = (('a', '1'), ('b', '2')) - url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0] - fs = _qs(self.request_class(url, method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) - self.assertEqual(fs[b'b'], [b'2']) - - #only query in url - data = {'a' : '1', 'b' : '2'} - fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) - self.assertEqual(fs[b'b'], [b'2']) - - #only fragment in url - data = (('a', '1'), ('b', '2')) - url = self.request_class('http://www.example.com/#fragment', method='GET', formdata=data).url.split('#')[0] + url = self.request_class('http://www.example.com/?a=0&a=2&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) + self.assertEqual(fs[b'c'], [b'3']) #None of both in url data = {'a' : '1', 'b' : '2'} From a9f68acb6dd7f5aca2bee95353278e5a40c32b8b Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Thu, 17 Jan 2019 23:51:09 +0530 Subject: [PATCH 012/568] modified code --- scrapy/http/request/form.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index fa5110382..2305491e2 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -41,15 +41,14 @@ class FormRequest(Request): if url_split.query: queries = (url_split.query).split('&') query_dict = {} - for x in queries: - query = x.split('=') - if formdata_key_list.count(query[0])==0: - query_dict[query[0]] = query[1] + k = x.split('=')[0] + v = x.split('=')[1] + if formdata_key_list.count(k)==0: + query_dict[k] = v query_str = '' - for k, v in query_dict: + for k, v in query_dict.items(): query_str += (k + '=' + v + '&') - self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else '')) else: self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else '')) From 7dee841b8b904a3961481f7b1999d819d2c17c5d Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sat, 19 Jan 2019 13:20:01 +0530 Subject: [PATCH 013/568] Update form.py --- scrapy/http/request/form.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2305491e2..af4ed1793 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ import six -from six.moves.urllib.parse import urljoin, urlencode, urlsplit +from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl import lxml.html from parsel.selector import create_root_node @@ -37,21 +37,10 @@ class FormRequest(Request): formdata_key_list = [] for k in querystr.split('&'): formdata_key_list.append(k.split('=')[0]) - - if url_split.query: - queries = (url_split.query).split('&') - query_dict = {} - for x in queries: - k = x.split('=')[0] - v = x.split('=')[1] - if formdata_key_list.count(k)==0: - query_dict[k] = v - query_str = '' - for k, v in query_dict.items(): - query_str += (k + '=' + v + '&') - self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else '')) - else: - self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else '')) + items = [] + items += [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list] + query_str = _urlencode(items, self.encoding) + self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else ''))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 722a30ac2bc35bdc7abb52d10417ed1191a3aee0 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sat, 19 Jan 2019 13:20:05 +0530 Subject: [PATCH 014/568] Update test_http_request.py --- tests/test_http_request.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a17acf93b..955db9e76 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -276,10 +276,10 @@ class FormRequestTest(RequestTest): def test_formdata_overrides_querystring_duplicates(self): #Both fragment and query in url - data = (('a', '1'), ('b', '2')) - url = self.request_class('http://www.example.com/?a=0&a=2&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] + data = (('a', 'one'), ('a', 'two'), ('b', '2')) + url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) self.assertEqual(fs[b'c'], [b'3']) From 5a55c4269d3389df4e486657caa58985f8e37e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Thu, 31 Jan 2019 17:20:29 -0200 Subject: [PATCH 015/568] Adding GCSFeedStorage --- scrapy/extensions/feedexport.py | 26 ++++++++++++++++++ tests/test_feedexport.py | 48 ++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 22ebf3b3f..00d5d8025 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -146,6 +146,32 @@ class S3FeedStorage(BlockingFeedStorage): key.close() + +class GCSFeedStorage(BlockingFeedStorage): + + project_id = None + bucket_name = None + blob_name = None + + def __init__(self, uri, project_id): + self.project_id = project_id + u = urlparse(uri) + self.bucket_name = u.hostname + self.blob_name = u.path[1:] # remove first "/" + + @classmethod + def from_crawler(cls, crawler, uri): + return cls(uri, crawler.settings['GCS_PROJECT_ID']) + + def _store_in_thread(self, file): + file.seek(0) + from google.cloud.storage import Client + client = Client(project=self.project_id) + bucket = client.get_bucket(self.bucket_name) + blob = bucket.blob(self.blob_name) + blob.upload_from_file(file) + + class FTPFeedStorage(BlockingFeedStorage): def __init__(self, uri): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e46c8c14e..f3c499b3f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -21,7 +21,7 @@ from w3lib.url import path_to_file_uri import scrapy from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( - IFeedStorage, FileFeedStorage, FTPFeedStorage, + IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage, S3FeedStorage, StdoutFeedStorage, BlockingFeedStorage) from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler @@ -187,6 +187,52 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(content, expected_content) +class GCSFeedStorageTest(unittest.TestCase): + + @mock.patch('scrapy.conf.settings', + new={'GCS_PROJECT_ID': 'conf_id' }, create=True) + def test_parse_settings(self): + try: + from google.cloud.storage import Client + except ImportError: + raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + + settings = {'GCS_PROJECT_ID': '123' } + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv') + assert storage.project_id == '123' + assert storage.bucket_name == 'mybucket' + assert storage.blob_name == 'export.csv' + + @defer.inlineCallbacks + def test_store(self): + try: + from google.cloud.storage import Client, Bucket, Blob + except ImportError: + raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + + uri = 'gcs://mybucket/export.csv' + project_id = 'myproject-123' + with mock.patch('google.cloud.storage.Client') as m: + client_mock = mock.create_autospec(Client) + m.return_value = client_mock + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + f = mock.Mock() + storage = GCSFeedStorage(uri, project_id) + yield storage.store(f) + + f.seek.assert_called_once_with(0) + m.assert_called_once_with(project=project_id) + client_mock.get_bucket.assert_called_once_with('mybucket') + bucket_mock.blob.assert_called_once_with('export.csv') + blob_mock.upload_from_file.assert_called_once_with(f) + class StdoutFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks From a4059851e7b6c8d712b2bc73dbff99be2d569d21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Thu, 31 Jan 2019 18:29:15 -0200 Subject: [PATCH 016/568] Refactoring tests --- scrapy/utils/test.py | 16 ++++++++++++++++ tests/test_feedexport.py | 15 +++++---------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4b935c51b..84eae97a7 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -7,6 +7,7 @@ import os from importlib import import_module from twisted.trial.unittest import SkipTest +from tests import mock from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore @@ -91,3 +92,18 @@ def assert_samelines(testcase, text1, text2, msg=None): line endings between platforms """ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) + +def mock_google_cloud_storage(): + """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob + classes and set their proper return values. + """ + from google.cloud.storage import Client, Bucket, Blob + client_mock = mock.create_autospec(Client) + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + return (client_mock, bucket_mock, blob_mock) \ No newline at end of file diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f3c499b3f..6d23c68c3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -24,7 +24,8 @@ from scrapy.extensions.feedexport import ( IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage, S3FeedStorage, StdoutFeedStorage, BlockingFeedStorage) -from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler +from scrapy.utils.test import (assert_aws_environ, get_s3_content_and_delete, + get_crawler, mock_google_cloud_storage) from scrapy.utils.python import to_native_str @@ -207,22 +208,16 @@ class GCSFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): try: - from google.cloud.storage import Client, Bucket, Blob + from google.cloud.storage import Client except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") uri = 'gcs://mybucket/export.csv' project_id = 'myproject-123' - with mock.patch('google.cloud.storage.Client') as m: - client_mock = mock.create_autospec(Client) + (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() + with mock.patch('google.cloud.storage.Client') as m: m.return_value = client_mock - bucket_mock = mock.create_autospec(Bucket) - client_mock.get_bucket.return_value = bucket_mock - - blob_mock = mock.create_autospec(Blob) - bucket_mock.blob.return_value = blob_mock - f = mock.Mock() storage = GCSFeedStorage(uri, project_id) yield storage.store(f) From 1bb6c4154c43d03d00147d810147c4f3c807505e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 09:04:01 -0200 Subject: [PATCH 017/568] Turning into instance attributes --- scrapy/extensions/feedexport.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 00d5d8025..a81f44045 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -149,10 +149,6 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - project_id = None - bucket_name = None - blob_name = None - def __init__(self, uri, project_id): self.project_id = project_id u = urlparse(uri) From fc6809b024dc25663432b6c0c5780021a827ea20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 09:08:54 -0200 Subject: [PATCH 018/568] Add gcs schema to FEED_STORAGES_BASE --- scrapy/settings/default_settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 3734a0a58..1a12f35a3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -145,6 +145,7 @@ FEED_STORAGES_BASE = { 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', + 'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage', } FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { From 4a53de165a53433a9a2a8cc4db34e5507c47fbd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 09:09:56 -0200 Subject: [PATCH 019/568] Sorted schemas alphabetically --- scrapy/extensions/feedexport.py | 1 - scrapy/settings/default_settings.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index a81f44045..8347b42ca 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -146,7 +146,6 @@ class S3FeedStorage(BlockingFeedStorage): key.close() - class GCSFeedStorage(BlockingFeedStorage): def __init__(self, uri, project_id): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 1a12f35a3..8769c01ba 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -142,10 +142,10 @@ FEED_STORAGES = {} FEED_STORAGES_BASE = { '': 'scrapy.extensions.feedexport.FileFeedStorage', 'file': 'scrapy.extensions.feedexport.FileFeedStorage', - 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', - 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', 'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage', + 's3': 'scrapy.extensions.feedexport.S3FeedStorage', + 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', } FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { From 2bbbd02bda368c2f052f3aa38f99498a632328bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 09:45:10 -0200 Subject: [PATCH 020/568] Adding an option to set ACL while uploading the blob to GCS --- scrapy/extensions/feedexport.py | 11 ++++++++--- scrapy/settings/default_settings.py | 2 ++ tests/test_feedexport.py | 10 ++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 8347b42ca..fbbf9bb97 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -148,15 +148,20 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri, project_id): + def __init__(self, uri, project_id, acl): self.project_id = project_id + self.acl = acl u = urlparse(uri) self.bucket_name = u.hostname self.blob_name = u.path[1:] # remove first "/" @classmethod def from_crawler(cls, crawler, uri): - return cls(uri, crawler.settings['GCS_PROJECT_ID']) + return cls( + uri, + crawler.settings['GCS_PROJECT_ID'], + crawler.settings['FEED_STORAGE_GCS_ACL'] + ) def _store_in_thread(self, file): file.seek(0) @@ -164,7 +169,7 @@ class GCSFeedStorage(BlockingFeedStorage): client = Client(project=self.project_id) bucket = client.get_bucket(self.bucket_name) blob = bucket.blob(self.blob_name) - blob.upload_from_file(file) + blob.upload_from_file(file, predefined_acl=self.acl) class FTPFeedStorage(BlockingFeedStorage): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8769c01ba..5d2862980 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,6 +159,8 @@ FEED_EXPORTERS_BASE = { } FEED_EXPORT_INDENT = 0 +FEED_STORAGE_GCS_ACL = None + FILES_STORE_S3_ACL = 'private' FILES_STORE_GCS_ACL = '' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 6d23c68c3..5cbca6d28 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -191,17 +191,18 @@ class S3FeedStorageTest(unittest.TestCase): class GCSFeedStorageTest(unittest.TestCase): @mock.patch('scrapy.conf.settings', - new={'GCS_PROJECT_ID': 'conf_id' }, create=True) + new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True) def test_parse_settings(self): try: from google.cloud.storage import Client except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - settings = {'GCS_PROJECT_ID': '123' } + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' } crawler = get_crawler(settings_dict=settings) storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv') assert storage.project_id == '123' + assert storage.acl == 'publicRead' assert storage.bucket_name == 'mybucket' assert storage.blob_name == 'export.csv' @@ -214,19 +215,20 @@ class GCSFeedStorageTest(unittest.TestCase): uri = 'gcs://mybucket/export.csv' project_id = 'myproject-123' + acl = 'publicRead' (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() with mock.patch('google.cloud.storage.Client') as m: m.return_value = client_mock f = mock.Mock() - storage = GCSFeedStorage(uri, project_id) + storage = GCSFeedStorage(uri, project_id, acl) yield storage.store(f) f.seek.assert_called_once_with(0) m.assert_called_once_with(project=project_id) client_mock.get_bucket.assert_called_once_with('mybucket') bucket_mock.blob.assert_called_once_with('export.csv') - blob_mock.upload_from_file.assert_called_once_with(f) + blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) class StdoutFeedStorageTest(unittest.TestCase): From cb5f800b0f7029d2bf2e09e9bc1391078c133a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 11:26:33 -0200 Subject: [PATCH 021/568] Adding documentation about Google Cloud Storage Feed Export --- docs/topics/feed-exports.rst | 22 ++++++++++++++++++++++ docs/topics/settings.rst | 18 ++++++++++++++++++ scrapy/settings/default_settings.py | 2 ++ 3 files changed, 42 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index b64dbfbfd..efb63b0ba 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -185,6 +185,27 @@ passed through the following settings: * :setting:`AWS_ACCESS_KEY_ID` * :setting:`AWS_SECRET_ACCESS_KEY` +.. _topics-feed-storage-gcs: + +Google Cloud Storage (GCS) +-------------------------- + +The feeds are stored on `Google Cloud Storage`_. + + * URI scheme: ``gcs`` + * Example URIs: + + * ``gcs://mybucket/path/to/export.csv`` + + * Required external libraries: `google-cloud-storage `_. + +For more information about authentication, please refer to `Google Cloud documentation `_. + +You can set a *Project ID* and *Access Control List (ACL)* through the following settings: + + * :setting:`FEED_STORAGE_GCS_ACL` + * :setting:`GCS_PROJECT_ID` + .. _topics-feed-storage-stdout: Standard output @@ -366,3 +387,4 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _Amazon S3: https://aws.amazon.com/s3/ .. _boto: https://github.com/boto/boto .. _botocore: https://github.com/boto/botocore +.. _Google Cloud Storage: https://cloud.google.com/storage/ \ No newline at end of file diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0ac26a9bd..90ae8fd93 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -749,6 +749,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler temporary files before uploading with :ref:`FTP feed storage ` and :ref:`Amazon S3 `. +.. setting:: FEED_STORAGE_GCS_ACL + +FEED_STORAGE_GCS_ACL +-------------------- + +The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. +For more information on how to set this value, please refer to `Google Cloud documentation `_. + .. setting:: FTP_PASSIVE_MODE FTP_PASSIVE_MODE @@ -786,6 +794,15 @@ Default: ``"anonymous"`` The username to use for FTP connections when there is no ``"ftp_user"`` in ``Request`` meta. +.. setting:: GCS_PROJECT_ID + +GCS_PROJECT_ID +----------------- + +Default: ``None`` + +The Project ID that will be used when storing data on `Google Cloud Storage`_. + .. setting:: ITEM_PIPELINES ITEM_PIPELINES @@ -1371,3 +1388,4 @@ case to see how to enable and use them. .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search +.. _Google Cloud Storage: https://cloud.google.com/storage/ \ No newline at end of file diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5d2862980..c17e94a64 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -168,6 +168,8 @@ FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' FTP_PASSIVE_MODE = True +GCS_PROJECT_ID = None + HTTPCACHE_ENABLED = False HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False From 0bb3d8ca93cf68e0ef231ee734f8a1a3c4075a72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Wed, 27 Feb 2019 18:41:01 -0300 Subject: [PATCH 022/568] Updating Google Cloud Storage scheme to gs instead of gcs --- docs/topics/feed-exports.rst | 4 ++-- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index efb63b0ba..0957a5997 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -192,10 +192,10 @@ Google Cloud Storage (GCS) The feeds are stored on `Google Cloud Storage`_. - * URI scheme: ``gcs`` + * URI scheme: ``gs`` * Example URIs: - * ``gcs://mybucket/path/to/export.csv`` + * ``gs://mybucket/path/to/export.csv`` * Required external libraries: `google-cloud-storage `_. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c17e94a64..50fcd1d0a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -143,7 +143,7 @@ FEED_STORAGES_BASE = { '': 'scrapy.extensions.feedexport.FileFeedStorage', 'file': 'scrapy.extensions.feedexport.FileFeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', - 'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage', + 'gs': 'scrapy.extensions.feedexport.GCSFeedStorage', 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', } diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 5cbca6d28..41df7d7af 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -200,7 +200,7 @@ class GCSFeedStorageTest(unittest.TestCase): settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' } crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv') + storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') assert storage.project_id == '123' assert storage.acl == 'publicRead' assert storage.bucket_name == 'mybucket' From 6eca6f92c6ae48bb136311308e77e82d7659cf43 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 14:59:34 +0530 Subject: [PATCH 023/568] Update form.py --- scrapy/http/request/form.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index af4ed1793..b6ca6ef9c 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -34,11 +34,8 @@ class FormRequest(Request): self._set_body(querystr) else: url_split = urlsplit(self.url) - formdata_key_list = [] - for k in querystr.split('&'): - formdata_key_list.append(k.split('=')[0]) - items = [] - items += [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list] + formdata_key_list = list(dict(parse_qsl(querystr)).keys()) + items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list] query_str = _urlencode(items, self.encoding) self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else ''))) From d75b61b96aad172e21c7d3aeba6f48419a63c72d Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 15:07:12 +0530 Subject: [PATCH 024/568] Update test_http_request.py --- tests/test_http_request.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 955db9e76..2da1cdf6d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -289,6 +289,12 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) + #GET duplicates are preserved + formdata=(('foo', 'bar'), ('foo', 'baz')) + fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data)) + self.assertEqual(set(fs[b'foo']), {b'bar', b'baz'}) + self.assertEqual(set(fs[b'a']), {b'1', b'2'}) + def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b'one': b'two', b'price': b'\xc2\xa3 100'} From fdf03a6d0daae79b9a3b421d19102873d5a75f46 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 15:12:44 +0530 Subject: [PATCH 025/568] correcting tests --- tests/test_http_request.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 2da1cdf6d..0c2f95262 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -289,10 +289,10 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) - #GET duplicates are preserved - formdata=(('foo', 'bar'), ('foo', 'baz')) + #Duplicate GET arguments are preserved + formdata={'foo' : 'bar'} fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data)) - self.assertEqual(set(fs[b'foo']), {b'bar', b'baz'}) + self.assertEqual(fs[b'foo'], [b'bar']) self.assertEqual(set(fs[b'a']), {b'1', b'2'}) def test_default_encoding_bytes(self): From 8831fafabc699bd2ab48e3e185712fbcd4f7cc7f Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 15:42:48 +0530 Subject: [PATCH 026/568] Update test_http_request.py --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 0c2f95262..5c30c9785 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -290,7 +290,7 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'b'], [b'2']) #Duplicate GET arguments are preserved - formdata={'foo' : 'bar'} + data={'foo' : 'bar'} fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data)) self.assertEqual(fs[b'foo'], [b'bar']) self.assertEqual(set(fs[b'a']), {b'1', b'2'}) From 7da460b7935940c93d5454019ea99f9f117b3e7b Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 17:25:15 +0530 Subject: [PATCH 027/568] Update form.py --- scrapy/http/request/form.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b6ca6ef9c..b44d7b9a6 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ import six -from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl +from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl, urlunsplit import lxml.html from parsel.selector import create_root_node @@ -34,10 +34,10 @@ class FormRequest(Request): self._set_body(querystr) else: url_split = urlsplit(self.url) - formdata_key_list = list(dict(parse_qsl(querystr)).keys()) - items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list] + formdata_keys = set(dict(parse_qsl(querystr)).keys()) + items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys] query_str = _urlencode(items, self.encoding) - self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else ''))) + self._set_url(urlunsplit((url_split.scheme, url_split.netloc, url_split.path, (query_str + '&' + querystr) if query_str else querystr, url_split.fragment))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From f7bf3abbd03a10124ed648d45abbb6c8eb3218b7 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Wed, 6 Mar 2019 14:10:03 +0530 Subject: [PATCH 028/568] Modified code --- scrapy/http/request/form.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b44d7b9a6..a0e7f76a9 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -37,7 +37,8 @@ class FormRequest(Request): formdata_keys = set(dict(parse_qsl(querystr)).keys()) items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys] query_str = _urlencode(items, self.encoding) - self._set_url(urlunsplit((url_split.scheme, url_split.netloc, url_split.path, (query_str + '&' + querystr) if query_str else querystr, url_split.fragment))) + query = (query_str + '&' + querystr) if query_str else querystr + self._set_url(urlunsplit(url_split._replace(query = query))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 120007c0577f819a26ff795ac8b9cd6fc397df19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 13:53:47 +0100 Subject: [PATCH 029/568] Add a FAQ entry on how to deal with long lists of allowed domains --- docs/faq.rst | 35 +++++++++++++++++++++++++++++++ docs/topics/spider-middleware.rst | 2 ++ 2 files changed, 37 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 7a0628f88..f56d26c0a 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,6 +149,41 @@ How can I make Scrapy consume less memory? See previous question. +How can I prevent memory errors due to many allowed domains? +------------------------------------------------------------ + +If you have a spider with a long list of +:attr:`~scrapy.spiders.Spider.allowed_domains` (e.g. 50,000+), consider +replacing the default +:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware +with a :ref:`custom spider middleware ` that requires +less memory. For example: + +- If your domain names are similar enough, use your own regular expression + instead joining the strings in + :attr:`~scrapy.spiders.Spider.allowed_domains` into a complex regular + expression. + +- If you can `meet the installation requirements`_, use pyre2_ instead of + Python’s re_ to compile your URL-filtering regular expression. See + :issue:`1908`. + +See also other suggestions at `StackOverflow`_. + +.. note:: Remember to disable + :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable + your custom implementation:: + + SPIDER_MIDDLEWARES = { + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, + 'myproject.middlewares.CustomOffsiteMiddleware': 500, + } + +.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation +.. _pyre2: https://github.com/andreasvc/pyre2 +.. _re: https://docs.python.org/library/re.html +.. _StackOverflow: https://stackoverflow.com/q/36440681/939364 + Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b7e42771..80357a987 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -54,6 +54,8 @@ value. For example, if you want to disable the off-site middleware:: Finally, keep in mind that some middlewares may need to be enabled through a particular setting. See each middleware documentation for more info. +.. _custom-spider-middleware: + Writing your own spider middleware ================================== From 35f7595dbebe7012c8f66336b76625b089a1fad3 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 11 Mar 2019 23:58:37 +0530 Subject: [PATCH 030/568] changed variable names --- scrapy/http/request/form.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a0e7f76a9..9af2db5ff 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -28,16 +28,16 @@ class FormRequest(Request): if formdata: items = formdata.items() if isinstance(formdata, dict) else formdata - querystr = _urlencode(items, self.encoding) + form_query_str = _urlencode(items, self.encoding) if self.method == 'POST': self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') - self._set_body(querystr) + self._set_body(form_query_str) else: url_split = urlsplit(self.url) - formdata_keys = set(dict(parse_qsl(querystr)).keys()) + formdata_keys = set(dict(parse_qsl(form_query_str)).keys()) items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys] - query_str = _urlencode(items, self.encoding) - query = (query_str + '&' + querystr) if query_str else querystr + url_query_str = _urlencode(items, self.encoding) + query = (url_query_str + '&' + form_query_str) if url_query_str else form_query_str self._set_url(urlunsplit(url_split._replace(query = query))) @classmethod From 282f24c510bdc6fff0e86ca16b7d681617af25ca Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Wed, 20 Mar 2019 18:46:22 +0530 Subject: [PATCH 031/568] Update form.py --- scrapy/http/request/form.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 9af2db5ff..03692f063 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ import six -from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl, urlunsplit +from six.moves.urllib.parse import urljoin, urlencode, urlsplit, urlunsplit import lxml.html from parsel.selector import create_root_node @@ -33,12 +33,7 @@ class FormRequest(Request): self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') self._set_body(form_query_str) else: - url_split = urlsplit(self.url) - formdata_keys = set(dict(parse_qsl(form_query_str)).keys()) - items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys] - url_query_str = _urlencode(items, self.encoding) - query = (url_query_str + '&' + form_query_str) if url_query_str else form_query_str - self._set_url(urlunsplit(url_split._replace(query = query))) + self._set_url(urlunsplit(urlsplit(self.url)._replace(query = form_query_str))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 4c89e53e684987c61857ef810374101440d5616c Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Wed, 20 Mar 2019 18:46:25 +0530 Subject: [PATCH 032/568] Update test_http_request.py --- tests/test_http_request.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5c30c9785..feea54a54 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -275,26 +275,17 @@ class FormRequestTest(RequestTest): self.assertEqual(r1.body, b'') def test_formdata_overrides_querystring_duplicates(self): - #Both fragment and query in url data = (('a', 'one'), ('a', 'two'), ('b', '2')) url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) - self.assertEqual(fs[b'c'], [b'3']) - #None of both in url data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) - #Duplicate GET arguments are preserved - data={'foo' : 'bar'} - fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data)) - self.assertEqual(fs[b'foo'], [b'bar']) - self.assertEqual(set(fs[b'a']), {b'1', b'2'}) - def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b'one': b'two', b'price': b'\xc2\xa3 100'} From 2cb4dc32052c306568206f0997de4b4e53069efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 22 Mar 2019 09:50:11 -0300 Subject: [PATCH 033/568] Mentioning to use JSON API for ACLs --- docs/topics/settings.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 90ae8fd93..fcdf31cac 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -755,7 +755,7 @@ FEED_STORAGE_GCS_ACL -------------------- The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. -For more information on how to set this value, please refer to `Google Cloud documentation `_. +For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_. .. setting:: FTP_PASSIVE_MODE @@ -1388,4 +1388,4 @@ case to see how to enable and use them. .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search -.. _Google Cloud Storage: https://cloud.google.com/storage/ \ No newline at end of file +.. _Google Cloud Storage: https://cloud.google.com/storage/ From dc8310e2929d228a224f12055a26cfef71d751cd Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Tue, 26 Mar 2019 15:42:58 +0530 Subject: [PATCH 034/568] changed tests --- tests/test_http_request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index feea54a54..81c1a4a9e 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -274,12 +274,13 @@ class FormRequestTest(RequestTest): r1 = self.request_class("http://www.example.com", formdata={}) self.assertEqual(r1.body, b'') - def test_formdata_overrides_querystring_duplicates(self): + def test_formdata_overrides_querystring(self): data = (('a', 'one'), ('a', 'two'), ('b', '2')) url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) + self.assertNone(fs[b'c']) data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) From 213b9eb879c4c6d8ae60ffa950688a38951b3349 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Tue, 26 Mar 2019 15:59:38 +0530 Subject: [PATCH 035/568] Update test_http_request.py --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 81c1a4a9e..56c7d3d94 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -280,7 +280,7 @@ class FormRequestTest(RequestTest): fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) - self.assertNone(fs[b'c']) + self.assertIsNone(fs[b'c']) data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) From ae856e8ba835d1d99510e9509a462b55a757e85f Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Tue, 26 Mar 2019 16:21:52 +0530 Subject: [PATCH 036/568] corrected tests --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 56c7d3d94..55fa4ad22 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -280,7 +280,7 @@ class FormRequestTest(RequestTest): fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) - self.assertIsNone(fs[b'c']) + self.assertIsNone(fs.get([b'c'])) data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) From 5f2ad5377e54ecdb3059891db4887eb09e56bfc6 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Tue, 26 Mar 2019 16:46:15 +0530 Subject: [PATCH 037/568] fixed typo --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 55fa4ad22..89e4a8683 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -280,7 +280,7 @@ class FormRequestTest(RequestTest): fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) - self.assertIsNone(fs.get([b'c'])) + self.assertIsNone(fs.get(b'c')) data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) From 110bc92e6b9c9c3ce1775a9ca1487df42a80d219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Thu, 29 Aug 2019 11:10:00 -0300 Subject: [PATCH 038/568] Fix default value of FEED_STORAGE_GCS_ACL --- scrapy/extensions/feedexport.py | 2 +- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fbbf9bb97..1e982c684 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -160,7 +160,7 @@ class GCSFeedStorage(BlockingFeedStorage): return cls( uri, crawler.settings['GCS_PROJECT_ID'], - crawler.settings['FEED_STORAGE_GCS_ACL'] + crawler.settings['FEED_STORAGE_GCS_ACL'] or None ) def _store_in_thread(self, file): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 50fcd1d0a..45257a61c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,7 +159,7 @@ FEED_EXPORTERS_BASE = { } FEED_EXPORT_INDENT = 0 -FEED_STORAGE_GCS_ACL = None +FEED_STORAGE_GCS_ACL = '' FILES_STORE_S3_ACL = 'private' FILES_STORE_GCS_ACL = '' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 41df7d7af..69f144d07 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -206,6 +206,24 @@ class GCSFeedStorageTest(unittest.TestCase): assert storage.bucket_name == 'mybucket' assert storage.blob_name == 'export.csv' + @mock.patch('scrapy.conf.settings', + new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': '' }, create=True) + def test_parse_empty_acl(self): + try: + from google.cloud.storage import Client + except ImportError: + raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': '' } + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') + assert storage.acl is None + + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None } + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') + assert storage.acl is None + @defer.inlineCallbacks def test_store(self): try: From ad6075440c0285d903ec7238354d093ea300b8f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Oct 2019 19:00:03 +0200 Subject: [PATCH 039/568] Fix references to Python types in parameter type fields --- docs/topics/contracts.rst | 2 +- docs/topics/email.rst | 10 ++++----- docs/topics/exporters.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/link-extractors.rst | 14 ++++++------- docs/topics/loaders.rst | 6 +++--- docs/topics/request-response.rst | 36 ++++++++++++++++---------------- scrapy/crawler.py | 2 +- scrapy/robotstxt.py | 4 ++-- scrapy/settings/__init__.py | 33 ++++++++++++----------------- scrapy/signalmanager.py | 2 +- 11 files changed, 53 insertions(+), 60 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 62f9a743b..15443f4cc 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -85,7 +85,7 @@ override three methods: .. class:: Contract(method, \*args) :param method: callback function to which the contract is associated - :type method: function + :type method: collections.abc.Callable :param args: list of arguments passed into the docstring (whitespace separated) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 949cdc638..73b1bdc3b 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -63,10 +63,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. :type smtpport: int :param smtptls: enforce using SMTP STARTTLS - :type smtptls: boolean + :type smtptls: bool :param smtpssl: enforce using a secure SSL connection - :type smtpssl: boolean + :type smtpssl: bool .. classmethod:: from_settings(settings) @@ -81,13 +81,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. :param to: the e-mail recipients - :type to: str or list of str + :type to: str or list :param subject: the subject of the e-mail :type subject: str :param cc: the e-mails to CC - :type cc: str or list of str + :type cc: str or list :param body: the e-mail body :type body: str @@ -97,7 +97,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. appear on the e-mail's attachment, ``mimetype`` is the mimetype of the attachment and ``file_object`` is a readable file object with the contents of the attachment - :type attachs: iterable + :type attachs: collections.abc.Iterable :param mimetype: the MIME type of the e-mail :type mimetype: str diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index a698a6a4e..da304922d 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -300,7 +300,7 @@ CsvItemExporter :param include_headers_line: If enabled, makes the exporter output a header line with the field names taken from :attr:`BaseItemExporter.fields_to_export` or the first exported item fields. - :type include_headers_line: boolean + :type include_headers_line: bool :param join_multivalued: The char (or chars) that will be used for joining multi-valued fields, if found. diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 8278e9849..657f1cc61 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -179,7 +179,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. :param ignore: if given, all objects from the specified class (or tuple of classes) will be ignored. - :type ignore: class or classes tuple + :type ignore: type or tuple .. function:: get_oldest(class_name) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 713a94e10..13b9ad7a5 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -59,13 +59,13 @@ LxmlLinkExtractor :param allow: a single regular expression (or list of regular expressions) that the (absolute) urls must match in order to be extracted. If not given (or empty), it will match all links. - :type allow: a regular expression (or list of) + :type allow: str or list :param deny: a single regular expression (or list of regular expressions) that the (absolute) urls must match in order to be excluded (ie. not extracted). It has precedence over the ``allow`` parameter. If not given (or empty) it won't exclude any links. - :type deny: a regular expression (or list of) + :type deny: str or list :param allow_domains: a single value or a list of string containing domains which will be considered for extracting the links @@ -97,7 +97,7 @@ LxmlLinkExtractor that the link's text must match in order to be extracted. If not given (or empty), it will match all links. If a list of regular expressions is given, the link will be extracted if it matches at least one. - :type restrict_text: a regular expression (or list of) + :type restrict_text: str or list :param tags: a tag or a list of tags to consider when extracting links. Defaults to ``('a', 'area')``. @@ -115,11 +115,11 @@ LxmlLinkExtractor different for requests with canonicalized and raw URLs. If you're using LinkExtractor to follow links it is more robust to keep the default ``canonicalize=False``. - :type canonicalize: boolean + :type canonicalize: bool :param unique: whether duplicate filtering should be applied to extracted links. - :type unique: boolean + :type unique: bool :param process_value: a function which receives each value extracted from the tag and attributes scanned and can modify the value and return a @@ -141,7 +141,7 @@ LxmlLinkExtractor if m: return m.group(1) - :type process_value: callable + :type process_value: collections.abc.Callable :param strip: whether to strip whitespaces from extracted attributes. According to HTML5 standard, leading and trailing whitespaces @@ -150,6 +150,6 @@ LxmlLinkExtractor elements, etc., so LinkExtractor strips space chars by default. Set ``strip=False`` to turn it off (e.g. if you're extracting urls from elements or attributes which allow leading/trailing whitespaces). - :type strip: boolean + :type strip: bool .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 1c2f1da4d..4137fdd24 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -320,7 +320,7 @@ ItemLoader objects :param re: a regular expression to use for extracting data from the given value using :meth:`~scrapy.utils.misc.extract_regex` method, applied before processors - :type re: str or compiled regex + :type re: str or typing.Pattern Examples:: @@ -365,7 +365,7 @@ ItemLoader objects :param re: a regular expression to use for extracting data from the selected XPath region - :type re: str or compiled regex + :type re: str or typing.Pattern Examples:: @@ -408,7 +408,7 @@ ItemLoader objects :param re: a regular expression to use for extracting data from the selected CSS region - :type re: str or compiled regex + :type re: str or typing.Pattern Examples:: diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 727c67482..2f99a72f5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -31,7 +31,7 @@ Request objects a :class:`Response`. :param url: the URL of this request - :type url: string + :type url: str :param callback: the function that will be called with the response of this request (once its downloaded) as its first parameter. For more information @@ -40,10 +40,10 @@ Request objects :meth:`~scrapy.spiders.Spider.parse` method will be used. Note that if exceptions are raised during processing, errback is called instead. - :type callback: callable + :type callback: collections.abc.Callable :param method: the HTTP method of this request. Defaults to ``'GET'``. - :type method: string + :type method: str :param meta: the initial values for the :attr:`Request.meta` attribute. If given, the dict passed in this parameter will be shallow copied. @@ -54,7 +54,7 @@ Request objects ``body`` is not given, an empty string is stored. Regardless of the type of this argument, the final value stored will be a ``str`` (never ``unicode`` or ``None``). - :type body: str or unicode + :type body: str :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If @@ -105,7 +105,7 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the body to ``str`` (if given as ``unicode``). - :type encoding: string + :type encoding: str :param priority: the priority of this request (defaults to ``0``). The priority is used by the scheduler to define the order used to process @@ -117,7 +117,7 @@ Request objects the scheduler. This is used when you want to perform an identical request multiple times, to ignore the duplicates filter. Use it with care, or you will get into crawling loops. Default to ``False``. - :type dont_filter: boolean + :type dont_filter: bool :param errback: a function that will be called if any exception was raised while processing the request. This includes pages that failed @@ -125,7 +125,7 @@ Request objects as first parameter. For more information, see :ref:`topics-request-response-ref-errbacks` below. - :type errback: callable + :type errback: collections.abc.Callable :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list @@ -407,7 +407,7 @@ fields with form data from :class:`Response` objects. :param formdata: is a dictionary (or iterable of (key, value) tuples) containing HTML Form data which will be url-encoded and assigned to the body of the request. - :type formdata: dict or iterable of tuples + :type formdata: dict or collections.abc.Iterable The :class:`FormRequest` objects support the following class method in addition to the standard :class:`Request` methods: @@ -439,20 +439,20 @@ fields with form data from :class:`Response` objects. :type response: :class:`Response` object :param formname: if given, the form with name attribute set to this value will be used. - :type formname: string + :type formname: str :param formid: if given, the form with id attribute set to this value will be used. - :type formid: string + :type formid: str :param formxpath: if given, the first form that matches the xpath will be used. - :type formxpath: string + :type formxpath: str :param formcss: if given, the first form that matches the css selector will be used. - :type formcss: string + :type formcss: str :param formnumber: the number of form to use, when the response contains multiple forms. The first one (and also the default) is ``0``. - :type formnumber: integer + :type formnumber: int :param formdata: fields to override in the form data. If a field was already present in the response ``
`` element, its value is @@ -470,7 +470,7 @@ fields with form data from :class:`Response` objects. :param dont_click: If True, the form data will be submitted without clicking in any element. - :type dont_click: boolean + :type dont_click: bool The other parameters of this class method are passed directly to the :class:`FormRequest` constructor. @@ -558,7 +558,7 @@ dealing with JSON requests. if :attr:`Request.body` argument is provided this parameter will be ignored. if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be set to ``'POST'`` automatically. - :type data: JSON serializable object + :type data: object :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data into JSON format. @@ -587,10 +587,10 @@ Response objects downloaded (by the Downloader) and fed to the Spiders for processing. :param url: the URL of this response - :type url: string + :type url: str :param status: the HTTP status of the response. Defaults to ``200``. - :type status: integer + :type status: int :param headers: the headers of this response. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). @@ -730,7 +730,7 @@ TextResponse objects body, it will be encoded using this encoding (remember the body attribute is always a string). If ``encoding`` is ``None`` (default value), the encoding will be looked up in the response headers and body instead. - :type encoding: string + :type encoding: str :class:`TextResponse` objects support the following attributes in addition to the standard :class:`Response` ones: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ded3c082b..84acf543f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -292,7 +292,7 @@ class CrawlerProcess(CrawlerRunner): If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. - :param boolean stop_after_crawl: stop or not the reactor when all + :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished """ if stop_after_crawl: diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 189f165d1..7faad308a 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -43,10 +43,10 @@ class RobotParser(with_metaclass(ABCMeta)): """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. :param url: Absolute URL - :type url: string + :type url: str :param user_agent: User agent - :type user_agent: string + :type user_agent: str """ pass diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index f28c7940d..95c02021e 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -105,10 +105,9 @@ class BaseSettings(MutableMapping): Get a setting value without affecting its original type. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ return self[name] if self[name] is not None else default @@ -123,10 +122,9 @@ class BaseSettings(MutableMapping): ``'0'`` will return ``False`` when using this method. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ got = self.get(name, default) try: @@ -145,10 +143,9 @@ class BaseSettings(MutableMapping): Get a setting value as an int. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ return int(self.get(name, default)) @@ -157,10 +154,9 @@ class BaseSettings(MutableMapping): Get a setting value as a float. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ return float(self.get(name, default)) @@ -173,10 +169,9 @@ class BaseSettings(MutableMapping): ``'one,two'`` will return a list ['one', 'two'] when using this method. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ value = self.get(name, default or []) if isinstance(value, six.string_types): @@ -194,10 +189,9 @@ class BaseSettings(MutableMapping): and losing all information about priority and mutability. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ value = self.get(name, default or {}) if isinstance(value, six.string_types): @@ -209,7 +203,7 @@ class BaseSettings(MutableMapping): counterpart. :param name: name of the dictionary-like setting - :type name: string + :type name: str """ compbs = BaseSettings() compbs.update(self[name + '_BASE']) @@ -222,7 +216,7 @@ class BaseSettings(MutableMapping): the given ``name`` does not exist. :param name: the setting name - :type name: string + :type name: str """ if name not in self: return None @@ -252,14 +246,13 @@ class BaseSettings(MutableMapping): otherwise they won't have any effect. :param name: the setting name - :type name: string + :type name: str :param value: the value to associate with the setting - :type value: any :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() priority = get_settings_priority(priority) @@ -283,11 +276,11 @@ class BaseSettings(MutableMapping): uppercase variable of ``module`` with the provided ``priority``. :param module: the module or the path of the module - :type module: module object or string + :type module: types.ModuleType or str :param priority: the priority of the settings. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() if isinstance(module, six.string_types): @@ -316,7 +309,7 @@ class BaseSettings(MutableMapping): :param priority: the priority of the settings. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() if isinstance(values, six.string_types): diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 296d27ed8..c24b16fcb 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -17,7 +17,7 @@ class SignalManager(object): section. :param receiver: the function to be connected - :type receiver: callable + :type receiver: collections.abc.Callable :param signal: the signal to connect to :type signal: object From 5479e7ecc7d30424dd2f3d9bbfb18abca765be92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Oct 2019 15:24:44 +0200 Subject: [PATCH 040/568] Indicate that lists of emails may be provided as a single string or as a list of strings --- docs/topics/email.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 73b1bdc3b..284849c56 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -80,13 +80,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. - :param to: the e-mail recipients + :param to: the e-mail recipients as a string or as a list of strings :type to: str or list :param subject: the subject of the e-mail :type subject: str - :param cc: the e-mails to CC + :param cc: the e-mails to CC as a string or as a list of strings :type cc: str or list :param body: the e-mail body From d96b9f860b01256a84bda641b190b423eb4910b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Oct 2019 15:24:59 +0200 Subject: [PATCH 041/568] Use object as type for parameters that allow any value --- scrapy/settings/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 95c02021e..d1a5093a6 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -108,6 +108,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ return self[name] if self[name] is not None else default @@ -125,6 +126,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ got = self.get(name, default) try: @@ -146,6 +148,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ return int(self.get(name, default)) @@ -157,6 +160,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ return float(self.get(name, default)) @@ -172,6 +176,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ value = self.get(name, default or []) if isinstance(value, six.string_types): @@ -192,6 +197,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ value = self.get(name, default or {}) if isinstance(value, six.string_types): @@ -249,6 +255,7 @@ class BaseSettings(MutableMapping): :type name: str :param value: the value to associate with the setting + :type default: object :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer From 0946eb335a285e1f210ba1185a564699f53b17d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Nov 2019 17:56:21 +0100 Subject: [PATCH 042/568] =?UTF-8?q?Port=20code=20from=20Twisted=E2=80=99s?= =?UTF-8?q?=20deprecated=20HTTPClientFactory=20into=20ScrapyHTTPClientFact?= =?UTF-8?q?ory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/core/downloader/webclient.py | 90 ++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 20 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 3fe13414a..16fd214a3 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,9 +1,9 @@ from time import time from six.moves.urllib.parse import urlparse, urlunparse, urldefrag -from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient -from twisted.internet import defer +from twisted.internet import defer, reactor +from twisted.internet.protocol import ClientFactory from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached @@ -93,18 +93,30 @@ class ScrapyHTTPPageGetter(HTTPClient): (self.factory.url, self.factory.timeout))) -class ScrapyHTTPClientFactory(HTTPClientFactory): - """Scrapy implementation of the HTTPClientFactory overwriting the - setUrl method to make use of our Url object that cache the parse - result. - """ +class ScrapyHTTPClientFactory(ClientFactory): protocol = ScrapyHTTPPageGetter + waiting = 1 noisy = False followRedirect = False afterFoundGet = False + def _build_response(self, body, request): + request.meta['download_latency'] = self.headers_time-self.start_time + status = int(self.status) + headers = Headers(self.response_headers) + respcls = responsetypes.from_args(headers=headers, url=self._url) + return respcls(url=self._url, status=status, headers=headers, body=body) + + def _set_connection_attributes(self, request): + parsed = urlparse_cached(request) + self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed) + proxy = request.meta.get('proxy') + if proxy: + self.scheme, _, self.host, self.port, _ = _parse(proxy) + self.path = self.url + def __init__(self, request, timeout=180): self._url = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface @@ -139,21 +151,59 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): elif self.method == b'POST': self.headers['Content-Length'] = 0 - def _build_response(self, body, request): - request.meta['download_latency'] = self.headers_time-self.start_time - status = int(self.status) - headers = Headers(self.response_headers) - respcls = responsetypes.from_args(headers=headers, url=self._url) - return respcls(url=self._url, status=status, headers=headers, body=body) + def __repr__(self): + return "<%s: %s>" % (self.__class__.__name__, self.url) - def _set_connection_attributes(self, request): - parsed = urlparse_cached(request) - self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed) - proxy = request.meta.get('proxy') - if proxy: - self.scheme, _, self.host, self.port, _ = _parse(proxy) - self.path = self.url + def _cancelTimeout(self, result, timeoutCall): + if timeoutCall.active(): + timeoutCall.cancel() + return result + + def buildProtocol(self, addr): + p = ClientFactory.buildProtocol(self, addr) + p.followRedirect = self.followRedirect + p.afterFoundGet = self.afterFoundGet + if self.timeout: + timeoutCall = reactor.callLater(self.timeout, p.timeout) + self.deferred.addBoth(self._cancelTimeout, timeoutCall) + return p def gotHeaders(self, headers): self.headers_time = time() self.response_headers = headers + + def gotStatus(self, version, status, message): + """ + Set the status of the request on us. + @param version: The HTTP version. + @type version: L{bytes} + @param status: The HTTP status code, an integer represented as a + bytestring. + @type status: L{bytes} + @param message: The HTTP status message. + @type message: L{bytes} + """ + self.version, self.status, self.message = version, status, message + + def page(self, page): + if self.waiting: + self.waiting = 0 + self.deferred.callback(page) + + def noPage(self, reason): + if self.waiting: + self.waiting = 0 + self.deferred.errback(reason) + + def clientConnectionFailed(self, _, reason): + """ + When a connection attempt fails, the request cannot be issued. If no + result has yet been provided to the result Deferred, provide the + connection failure reason as an error result. + """ + if self.waiting: + self.waiting = 0 + # If the connection attempt failed, there is nothing more to + # disconnect, so just fire that Deferred now. + self._disconnectedDeferred.callback(None) + self.deferred.errback(reason) From 42954d0df9a78e6641996fbf2eb76afdef9c1856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 20 Nov 2019 08:16:33 +0100 Subject: [PATCH 043/568] Mention that ScrapyHTTPClientFactory has Twisted code --- scrapy/core/downloader/webclient.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 16fd214a3..26726afd4 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -93,6 +93,10 @@ class ScrapyHTTPPageGetter(HTTPClient): (self.factory.url, self.factory.timeout))) +# This class used to inherit from Twisted’s +# twisted.web.client.HTTPClientFactory. When that class was deprecated in +# Twisted (https://github.com/twisted/twisted/pull/643), we merged its +# non-overriden code into this class. class ScrapyHTTPClientFactory(ClientFactory): protocol = ScrapyHTTPPageGetter From b9a58798eed39bf543b9682d9ce43b13378a5074 Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 24 May 2014 20:24:01 +0000 Subject: [PATCH 044/568] change Scraper API to call internal `_parse` method A Spider class using internal pre-processing can have first dibs at this and then call a public `parse` method for subclass hooking. --- scrapy/core/scraper.py | 2 +- scrapy/spiders/__init__.py | 3 +++ scrapy/spiders/crawl.py | 2 +- scrapy/spiders/feed.py | 4 ++-- tests/test_spider.py | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 99114d3bb..ad6649134 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -145,7 +145,7 @@ class Scraper(object): def call_spider(self, result, request, spider): result.request = request dfd = defer_result(result) - callback = request.callback or spider.parse + callback = request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) warn_on_generator_with_return_value(spider, request.errback) dfd.addCallbacks(callback=callback, diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 9429f6cb2..1011eb870 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -80,6 +80,9 @@ class Spider(object_ref): """ This method is deprecated. """ return Request(url, dont_filter=True) + def _parse(self, response): + return self.parse(response) + def parse(self, response): raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index a2c364c0e..e28d17dcd 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -74,7 +74,7 @@ class CrawlSpider(Spider): super(CrawlSpider, self).__init__(*a, **kw) self._compile_rules() - def parse(self, response): + def _parse(self, response): return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True) def parse_start_url(self, response): diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index c566f0236..11bd17db4 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -61,7 +61,7 @@ class XMLFeedSpider(Spider): for result_item in self.process_results(response, ret): yield result_item - def parse(self, response): + def _parse(self, response): if not hasattr(self, 'parse_node'): raise NotConfigured('You must define parse_node method in order to scrape this XML feed') @@ -128,7 +128,7 @@ class CSVFeedSpider(Spider): for result_item in self.process_results(response, ret): yield result_item - def parse(self, response): + def _parse(self, response): if not hasattr(self, 'parse_row'): raise NotConfigured('You must define parse_row method in order to scrape this CSV feed') response = self.adapt_response(response) diff --git a/tests/test_spider.py b/tests/test_spider.py index 317a27076..6fbec7e58 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -142,7 +142,7 @@ class XMLFeedSpiderTest(SpiderTest): for iterator in ('iternodes', 'xml'): spider = _XMLSpider('example', iterator=iterator) - output = list(spider.parse(response)) + output = list(spider._parse(response)) self.assertEqual(len(output), 2, iterator) self.assertEqual(output, [ {'loc': [u'http://www.example.com/Special-Offers.html'], From 5982e3477c732f4dff5accea6eab486e4ab52c3e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Dec 2019 14:12:21 -0300 Subject: [PATCH 045/568] Take keyword arguments in base parsing methods --- docs/topics/spiders.rst | 12 +++++++----- scrapy/spiders/__init__.py | 6 +++--- scrapy/spiders/crawl.py | 11 ++++++++--- scrapy/spiders/feed.py | 4 ++-- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index b0fb14e24..dd763b607 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -362,12 +362,14 @@ CrawlSpider This spider also exposes an overrideable method: - .. method:: parse_start_url(response) + .. method:: parse_start_url(response, **kwargs) - This method is called for the start_urls responses. It allows to parse - the initial responses and must return either an - :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request` - object, or an iterable containing any of them. + This method is called for each response produced for the URLs in + the spider's ``start_urls`` attribute. It allows to parse + the initial responses and must return either an item + (:class:`scrapy.item.Item` or :class:`dict`), + a :class:`~scrapy.http.Request`, + or an iterable containing any of them. Crawling rules ~~~~~~~~~~~~~~ diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 1011eb870..3e19f1e23 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -80,10 +80,10 @@ class Spider(object_ref): """ This method is deprecated. """ return Request(url, dont_filter=True) - def _parse(self, response): - return self.parse(response) + def _parse(self, response, **kwargs): + return self.parse(response, **kwargs) - def parse(self, response): + def parse(self, response, **kwargs): raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) @classmethod diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index e28d17dcd..4ec0de78c 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -74,10 +74,15 @@ class CrawlSpider(Spider): super(CrawlSpider, self).__init__(*a, **kw) self._compile_rules() - def _parse(self, response): - return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True) + def _parse(self, response, **kwargs): + return self._parse_response( + response=response, + callback=self.parse_start_url, + cb_kwargs=kwargs, + follow=True, + ) - def parse_start_url(self, response): + def parse_start_url(self, response, **kwargs): return [] def process_results(self, response, results): diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 11bd17db4..4fa6009a5 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -61,7 +61,7 @@ class XMLFeedSpider(Spider): for result_item in self.process_results(response, ret): yield result_item - def _parse(self, response): + def _parse(self, response, **kwargs): if not hasattr(self, 'parse_node'): raise NotConfigured('You must define parse_node method in order to scrape this XML feed') @@ -128,7 +128,7 @@ class CSVFeedSpider(Spider): for result_item in self.process_results(response, ret): yield result_item - def _parse(self, response): + def _parse(self, response, **kwargs): if not hasattr(self, 'parse_row'): raise NotConfigured('You must define parse_row method in order to scrape this CSV feed') response = self.adapt_response(response) From 8d4948f6ca44a76ee7714c8b4b1c46ef73a8845e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 26 Dec 2019 14:38:11 -0300 Subject: [PATCH 046/568] [test] Override CrawlSpider.parse --- tests/spiders.py | 35 +++++++++++++++++++++++++++++------ tests/test_crawl.py | 31 +++++++++++++++++++++++++++---- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index 39c8da0b6..dcc475ca7 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -186,13 +186,39 @@ class DuplicateStartRequestsSpider(MockServerSpider): self.visited += 1 -class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider): - name = 'crawl_spider_with_errback' +class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): + """ + A CrawlSpider which overrides the 'parse' method + """ + name = 'crawl_spider_with_parse_method' custom_settings = { 'RETRY_HTTP_CODES': [], # no need to retry } rules = ( - Rule(LinkExtractor(), callback='callback', errback='errback', follow=True), + Rule(LinkExtractor(), callback='parse', follow=True), + ) + + def start_requests(self): + test_body = b""" + + Page title<title></head> + <body> + <p><a href="/status?n=200">Item 200</a></p> <!-- callback --> + <p><a href="/status?n=201">Item 201</a></p> <!-- callback --> + </body> + </html> + """ + url = self.mockserver.url("/alpayload") + yield Request(url, method="POST", body=test_body) + + def parse(self, response): + self.logger.info('[parse] status %i', response.status) + + +class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): + name = 'crawl_spider_with_errback' + rules = ( + Rule(LinkExtractor(), callback='parse', errback='errback', follow=True), ) def start_requests(self): @@ -211,8 +237,5 @@ class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider): url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=test_body) - def callback(self, response): - self.logger.info('[callback] status %i', response.status) - def errback(self, failure): self.logger.info('[errback] status %i', failure.value.response.status) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f433fcea6..4299e4bbb 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -9,8 +9,9 @@ from scrapy.crawler import CrawlerRunner from scrapy.http import Request from scrapy.utils.python import to_unicode from tests.mockserver import MockServer -from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider, - SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback) +from tests.spiders import (BrokenStartRequestsSpider, CrawlSpiderWithErrback, + CrawlSpiderWithParseMethod, DelaySpider, SimpleSpider, + DuplicateStartRequestsSpider, FollowAllSpider, SingleRequestSpider) class CrawlTestCase(TestCase): @@ -297,6 +298,27 @@ with multiples lines self._assert_retried(log) self.assertIn("Got response 200", str(log)) + +class CrawlSpiderTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self.runner = CrawlerRunner() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_crawlspider_with_parse(self): + self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver) + + with LogCapture() as log: + yield self.runner.join() + + self.assertIn("[parse] status 200", str(log)) + self.assertIn("[parse] status 201", str(log)) + @defer.inlineCallbacks def test_crawlspider_with_errback(self): self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver) @@ -304,7 +326,8 @@ with multiples lines with LogCapture() as log: yield self.runner.join() - self.assertIn("[callback] status 200", str(log)) - self.assertIn("[callback] status 201", str(log)) + self.assertIn("[parse] status 200", str(log)) + self.assertIn("[parse] status 201", str(log)) self.assertIn("[errback] status 404", str(log)) self.assertIn("[errback] status 500", str(log)) + self.assertIn("[errback] status 501", str(log)) From c54df8253a67bb6863a25bf7d667096adc73a040 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Thu, 26 Dec 2019 15:12:19 -0300 Subject: [PATCH 047/568] [test] Handle keyword args in CrawlSpider.parse --- tests/spiders.py | 5 +++-- tests/test_crawl.py | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index dcc475ca7..c042eb7fe 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -211,8 +211,9 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=test_body) - def parse(self, response): - self.logger.info('[parse] status %i', response.status) + def parse(self, response, foo=None): + self.logger.info('[parse] status %i (foo: %s)', response.status, foo) + yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"}) class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 4299e4bbb..6247ced35 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -316,8 +316,9 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield self.runner.join() - self.assertIn("[parse] status 200", str(log)) - self.assertIn("[parse] status 201", str(log)) + self.assertIn("[parse] status 200 (foo: None)", str(log)) + self.assertIn("[parse] status 201 (foo: None)", str(log)) + self.assertIn("[parse] status 202 (foo: bar)", str(log)) @defer.inlineCallbacks def test_crawlspider_with_errback(self): @@ -326,8 +327,9 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield self.runner.join() - self.assertIn("[parse] status 200", str(log)) - self.assertIn("[parse] status 201", str(log)) + self.assertIn("[parse] status 200 (foo: None)", str(log)) + self.assertIn("[parse] status 201 (foo: None)", str(log)) + self.assertIn("[parse] status 202 (foo: bar)", str(log)) self.assertIn("[errback] status 404", str(log)) self.assertIn("[errback] status 500", str(log)) self.assertIn("[errback] status 501", str(log)) From 8a1dc26d4662ec0c81d22c13cffa44255a7c29ca Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Thu, 26 Dec 2019 15:14:47 -0300 Subject: [PATCH 048/568] [doc] Note about the 'parse' method for CrawlSpider/XMLFeedSpider --- docs/topics/spiders.rst | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index dd763b607..406f50fb3 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -392,11 +392,6 @@ Crawling rules object will contain the text of the link that produced the :class:`~scrapy.http.Request` in its ``meta`` dictionary (under the ``link_text`` key) - .. warning:: When writing crawl spider rules, avoid using ``parse`` as - callback, since the :class:`CrawlSpider` uses the ``parse`` method - itself to implement its logic. So if you override the ``parse`` method, - the crawl spider will no longer work. - ``cb_kwargs`` is a dict containing the keyword arguments to be passed to the callback function. @@ -422,6 +417,12 @@ Crawling rules It receives a :class:`Twisted Failure <twisted.python.failure.Failure>` instance as first parameter. + +.. warning:: Because of its internal implementation, you must explicitly set + callbacks for new requests when writing :class:`CrawlSpider`-based spiders; + unexpected behaviour can occur otherwise. + + CrawlSpider example ~~~~~~~~~~~~~~~~~~~ @@ -452,6 +453,11 @@ Let's now take a look at an example CrawlSpider with rules:: item['name'] = response.xpath('//td[@id="item_name"]/text()').get() item['description'] = response.xpath('//td[@id="item_description"]/text()').get() item['link_text'] = response.meta['link_text'] + url = response.xpath('//td[@id="additional_data"]/@href').get() + return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item)) + + def parse_additional_page(self, response, item): + item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get() return item @@ -545,6 +551,11 @@ XMLFeedSpider those results. It must return a list of results (Items or Requests). +.. warning:: Because of its internal implementation, you must explicitly set + callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders; + unexpected behaviour can occur otherwise. + + XMLFeedSpider example ~~~~~~~~~~~~~~~~~~~~~ From 0641ba0faa97498ca5bee39c4e8faec58d5f0522 Mon Sep 17 00:00:00 2001 From: faizan2700 <syedfaizan824@gmail.com> Date: Sun, 2 Feb 2020 16:54:22 +0530 Subject: [PATCH 049/568] SCRAPY_CHECK will be set while running contact --- scrapy/commands/check.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 9d4437a47..09a76ca7a 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -78,19 +78,19 @@ class Command(ScrapyCommand): elif tested_methods: self.crawler_process.crawl(spidercls) - # start checks - if opts.list: - for spider, methods in sorted(contract_reqs.items()): - if not methods and not opts.verbose: - continue - print(spider) - for method in sorted(methods): - print(' * %s' % method) - else: - start = time.time() - self.crawler_process.start() - stop = time.time() + # start checks + if opts.list: + for spider, methods in sorted(contract_reqs.items()): + if not methods and not opts.verbose: + continue + print(spider) + for method in sorted(methods): + print(' * %s' % method) + else: + start = time.time() + self.crawler_process.start() + stop = time.time() - result.printErrors() - result.printSummary(start, stop) - self.exitcode = int(not result.wasSuccessful()) + result.printErrors() + result.printSummary(start, stop) + self.exitcode = int(not result.wasSuccessful()) From 7f2d3051feb0bc8f868a3eea8310e2fc8c461287 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Thu, 6 Feb 2020 18:19:40 -0300 Subject: [PATCH 050/568] Fix Flake8 issue --- tests/test_crawl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f1d502ff7..225fe7a0e 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -14,8 +14,8 @@ from tests.mockserver import MockServer from tests.spiders import ( AsyncDefAsyncioReturnSpider, AsyncDefAsyncioSpider, - AsyncDefSpider, - BrokenStartRequestsSpider, + AsyncDefSpider, + BrokenStartRequestsSpider, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, DelaySpider, From 7025c18b159f1ce14b5732dc7d262efbdc72cf5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 10 Feb 2020 19:43:23 +0100 Subject: [PATCH 051/568] Clear line of spaces --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 9f837221e..8dc08710f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -184,7 +184,7 @@ passed through the following settings: * :setting:`AWS_ACCESS_KEY_ID` * :setting:`AWS_SECRET_ACCESS_KEY` - + You can also define a custom ACL for exported feeds using this setting: * :setting:`FEED_STORAGE_S3_ACL` From 73e88d036c72aa628af50371aace4cfce7e286e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 12 Feb 2020 17:17:38 +0100 Subject: [PATCH 052/568] Import mock from unittest --- scrapy/utils/test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4ff6d73a5..00002c303 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,14 +2,13 @@ This module contains some assorted functions used in tests """ -from __future__ import absolute_import -from posixpath import split import asyncio import os +from posixpath import split +from unittest import mock from importlib import import_module from twisted.trial.unittest import SkipTest -from tests import mock from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore From e1be078eaa17a8df72716932fde07e225f79745f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 12 Feb 2020 17:38:06 +0100 Subject: [PATCH 053/568] Fix Flake8-reported issues --- scrapy/utils/test.py | 2 ++ tests/test_feedexport.py | 27 ++++++++++++++------------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 00002c303..7442a2f33 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -120,12 +120,14 @@ def assert_samelines(testcase, text1, text2, msg=None): """ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) + def get_from_asyncio_queue(value): q = asyncio.Queue() getter = q.get() q.put_nowait(value) return getter + def mock_google_cloud_storage(): """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d97b199fe..2b299503b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -25,9 +25,13 @@ from scrapy.extensions.feedexport import ( IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage, S3FeedStorage, StdoutFeedStorage, BlockingFeedStorage) -from scrapy.utils.test import (assert_aws_environ, get_s3_content_and_delete, - get_crawler, mock_google_cloud_storage) from scrapy.utils.python import to_unicode +from scrapy.utils.test import ( + assert_aws_environ, + get_s3_content_and_delete, + get_crawler, + mock_google_cloud_storage, +) class FileFeedStorageTest(unittest.TestCase): @@ -362,15 +366,13 @@ class S3FeedStorageTest(unittest.TestCase): class GCSFeedStorageTest(unittest.TestCase): - @mock.patch('scrapy.conf.settings', - new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True) def test_parse_settings(self): try: - from google.cloud.storage import Client + from google.cloud.storage import Client # noqa except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' } + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead'} crawler = get_crawler(settings_dict=settings) storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') assert storage.project_id == '123' @@ -378,20 +380,18 @@ class GCSFeedStorageTest(unittest.TestCase): assert storage.bucket_name == 'mybucket' assert storage.blob_name == 'export.csv' - @mock.patch('scrapy.conf.settings', - new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': '' }, create=True) def test_parse_empty_acl(self): try: - from google.cloud.storage import Client + from google.cloud.storage import Client # noqa except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': '' } + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': ''} crawler = get_crawler(settings_dict=settings) storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') assert storage.acl is None - settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None } + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None} crawler = get_crawler(settings_dict=settings) storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') assert storage.acl is None @@ -399,7 +399,7 @@ class GCSFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): try: - from google.cloud.storage import Client + from google.cloud.storage import Client # noqa except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") @@ -407,7 +407,7 @@ class GCSFeedStorageTest(unittest.TestCase): project_id = 'myproject-123' acl = 'publicRead' (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() - with mock.patch('google.cloud.storage.Client') as m: + with mock.patch('google.cloud.storage.Client') as m: m.return_value = client_mock f = mock.Mock() @@ -420,6 +420,7 @@ class GCSFeedStorageTest(unittest.TestCase): bucket_mock.blob.assert_called_once_with('export.csv') blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + class StdoutFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks From a175b6efc319ecbfd03b19705c25bd8bd44ca339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com> Date: Fri, 27 Mar 2020 02:10:10 -0300 Subject: [PATCH 054/568] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..9bc324c32 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,32 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + + +pool: + vmImage: 'windows-2019' +strategy: + matrix: + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + +- script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + +- script: | + pip install pytest pytest-azurepipelines + pytest + displayName: 'pytest' From 02206e5ffe74fe8107c272ad35920928544323b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com> Date: Fri, 27 Mar 2020 02:20:39 -0300 Subject: [PATCH 055/568] Run tox --- azure-pipelines.yml | 9 ++++++--- tests/requirements-py3.txt | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9bc324c32..489cfe53b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,10 +10,13 @@ strategy: matrix: Python35: python.version: '3.5' + TOXENV: py35 Python36: python.version: '3.6' + TOXENV: py36 Python37: python.version: '3.7' + TOXENV: py37 steps: - task: UsePythonVersion@0 @@ -27,6 +30,6 @@ steps: displayName: 'Install dependencies' - script: | - pip install pytest pytest-azurepipelines - pytest - displayName: 'pytest' + pip install -U tox twine wheel codecov + tox + displayName: 'Run test suite' diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index d207c5fb0..8896f4614 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -6,6 +6,7 @@ pytest < 5.4 pytest-cov pytest-twisted >= 1.11 pytest-xdist +pytest-azurepipelines sybil testfixtures From 0699e6bb1600ff943e131a3b7a299aa038145198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com> Date: Fri, 27 Mar 2020 02:22:05 -0300 Subject: [PATCH 056/568] no need to install requirements.txt --- azure-pipelines.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 489cfe53b..ffc4d549b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,11 +24,6 @@ steps: versionSpec: '$(python.version)' displayName: 'Use Python $(python.version)' -- script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - displayName: 'Install dependencies' - - script: | pip install -U tox twine wheel codecov tox From 3fb0027138ab44b16be6e11677626d69b8c90c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Sat, 28 Mar 2020 17:36:50 +0100 Subject: [PATCH 057/568] =?UTF-8?q?Require=20sybil=20=E2=89=A5=201.3.0=20f?= =?UTF-8?q?or=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index d207c5fb0..e7c86e0e9 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -6,7 +6,7 @@ pytest < 5.4 pytest-cov pytest-twisted >= 1.11 pytest-xdist -sybil +sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures # optional for shell wrapper tests From 2f510fd47d8d10217f7b18b531205dd8c252eaef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 15 Apr 2020 21:10:05 +0200 Subject: [PATCH 058/568] Fix ShellTest.test_local_file on Windows --- scrapy/utils/url.py | 13 +++++++------ tests/test_command_shell.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index c9abb12d5..c29ed4461 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -85,11 +85,9 @@ def add_http_if_no_scheme(url): def guess_scheme(url): - """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" - parts = urlparse(url) - if parts.scheme: - return url - # Note: this does not match Windows filepath + """Add an URL scheme if missing: file:// for filepath-like input or + http:// otherwise.""" + # POSIX path if re.match(r'''^ # start with... ( \. # ...a single dot, @@ -99,7 +97,10 @@ def guess_scheme(url): )? # optional match of ".", ".." or ".blabla" / # at least one "/" for a file path, . # and something after the "/" - ''', parts.path, flags=re.VERBOSE): + ''', url, flags=re.VERBOSE): + return any_to_uri(url) + # Windows drive-letter path + elif re.match(r'''^[a-z]:\\''', url, flags=re.IGNORECASE): return any_to_uri(url) else: return add_http_if_no_scheme(url) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index d664b6ade..acf8e9f71 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -94,7 +94,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_local_file(self): - filepath = join(tests_datadir, 'test_site/index.html') + filepath = join(tests_datadir, 'test_site', 'index.html') _, out, _ = yield self.execute([filepath, '-c', 'item']) assert b'{}' in out From e5b23f4b00962df76d8302ebf869ef4a4319e142 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Wed, 18 Mar 2020 11:26:59 +0300 Subject: [PATCH 059/568] fix #4250: add batch deliveries --- scrapy/extensions/feedexport.py | 54 +++++++++++++++++++++-------- scrapy/settings/default_settings.py | 1 + 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 998d2a5d1..906f99fee 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -241,6 +241,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') + self.storage_batch = self.settings.getint('FEED_STORAGE_BATCH') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -250,19 +251,7 @@ class FeedExporter: def open_spider(self, spider): for uri, feed in self.feeds.items(): uri = uri % self._get_uri_params(spider, feed['uri_params']) - storage = self._get_storage(uri) - file = storage.open(spider) - exporter = self._get_exporter( - file=file, - format=feed['format'], - fields_to_export=feed['fields'], - encoding=feed['encoding'], - indent=feed['indent'], - ) - slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty']) - self.slots.append(slot) - if slot.store_empty: - slot.start_exporting() + self.slots.append(self._start_new_batch(None, uri, feed, spider)) def close_spider(self, spider): deferred_list = [] @@ -285,11 +274,48 @@ class FeedExporter: deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None + def _start_new_batch(self, previous_batch_slot, uri, feed, spider): + """ + Redirect the output data stream to a new file. + Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero. + """ + if previous_batch_slot is not None: + previous_batch_slot.exporter.finish_exporting() + previous_batch_slot.storage.store(previous_batch_slot.file) + storage = self._get_storage(uri) + file = storage.open(spider) + exporter = self._get_exporter( + file=file, + format=feed['format'], + fields_to_export=feed['fields'], + encoding=feed['encoding'], + indent=feed['indent'] + ) + slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty']) + if slot.store_empty: + slot.start_exporting() + return slot + + def _get_uri_of_partial(self, slot, feed, spider): + """Get uri for each partial using datetime.now().isoformat()""" + uri = (slot.uri % self._get_uri_params(spider, feed['uri_params'])).split('.')[0] + '.' + uri = uri + datetime.now().isoformat() + '.' + feed['format'] + return uri + def item_scraped(self, item, spider): - for slot in self.slots: + slots = [] + for idx, slot in enumerate(self.slots): slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 + if self.storage_batch and slot.itemcount % self.storage_batch == 0: + uri = self._get_uri_of_partial(slot, self.feeds[slot.uri], spider) + slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider)) + self.feeds[uri] = self.feeds[slot.uri] + self.feeds.pop(slot.uri) + self.slots[idx] = None + self.slots = [slot for slot in self.slots if slot is not None] + self.slots.extend(slots) def _load_components(self, setting_prefix): conf = without_none_values(self.settings.getwithbase(setting_prefix)) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 077317c81..690e044c5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,6 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } +FEED_STORAGE_BATCH = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', From 8b4566ff93843cdf17ada069dc09261a99971d26 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Wed, 18 Mar 2020 14:21:21 +0300 Subject: [PATCH 060/568] fix wrong name of first file in partial deliveries --- scrapy/extensions/feedexport.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 906f99fee..4f7c6bf07 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -249,6 +249,8 @@ class FeedExporter: raise NotConfigured def open_spider(self, spider): + if self.storage_batch: + self.feeds = {self._get_uri_of_partial(uri, feed, spider): feed for uri, feed in self.feeds.items()} for uri, feed in self.feeds.items(): uri = uri % self._get_uri_params(spider, feed['uri_params']) self.slots.append(self._start_new_batch(None, uri, feed, spider)) @@ -296,11 +298,11 @@ class FeedExporter: slot.start_exporting() return slot - def _get_uri_of_partial(self, slot, feed, spider): + def _get_uri_of_partial(self, template_uri, feed, spider): """Get uri for each partial using datetime.now().isoformat()""" - uri = (slot.uri % self._get_uri_params(spider, feed['uri_params'])).split('.')[0] + '.' - uri = uri + datetime.now().isoformat() + '.' + feed['format'] - return uri + template_uri = (template_uri % self._get_uri_params(spider, feed['uri_params'])) + uri_name = template_uri.split('.')[0] + return '{}.{}.{}'.format(uri_name, datetime.now().isoformat(), feed["format"]) def item_scraped(self, item, spider): slots = [] @@ -309,11 +311,12 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 if self.storage_batch and slot.itemcount % self.storage_batch == 0: - uri = self._get_uri_of_partial(slot, self.feeds[slot.uri], spider) + uri = self._get_uri_of_partial(slot.uri, self.feeds[slot.uri], spider) slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider)) self.feeds[uri] = self.feeds[slot.uri] self.feeds.pop(slot.uri) self.slots[idx] = None + self.slots = [slot for slot in self.slots if slot is not None] self.slots.extend(slots) From 0723e3f4f9777a87d0df3b2e2fddfeac9099dd3b Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 19 Mar 2020 21:17:02 +0300 Subject: [PATCH 061/568] add batch_id, add error if uri is specified incorrectly --- scrapy/extensions/feedexport.py | 73 ++++++++++++++++++++--------- scrapy/settings/default_settings.py | 2 +- 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4f7c6bf07..38b25bf4a 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -180,14 +180,16 @@ class FTPFeedStorage(BlockingFeedStorage): class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty): + def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri): self.file = file self.exporter = exporter self.storage = storage # feed params - self.uri = uri + self.batch_id = batch_id self.format = format self.store_empty = store_empty + self.template_uri = template_uri + self.uri = uri # flags self.itemcount = 0 self._exporting = False @@ -241,19 +243,28 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch = self.settings.getint('FEED_STORAGE_BATCH') + self.storage_batch_size = self.settings.getint('FEED_STORAGE_BATCH_SIZE') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured + if not self._batch_deliveries_supported(uri): + raise NotConfigured if not self._exporter_supported(feed['format']): raise NotConfigured def open_spider(self, spider): - if self.storage_batch: - self.feeds = {self._get_uri_of_partial(uri, feed, spider): feed for uri, feed in self.feeds.items()} for uri, feed in self.feeds.items(): - uri = uri % self._get_uri_params(spider, feed['uri_params']) - self.slots.append(self._start_new_batch(None, uri, feed, spider)) + batch_id = 1 + uri_params = self._get_uri_params(spider, feed['uri_params']) + uri_params['batch_id'] = batch_id + self.slots.append(self._start_new_batch( + previous_batch_slot=None, + uri=uri % uri_params, + feed=feed, + spider=spider, + batch_id=batch_id, + template_uri=uri + )) def close_spider(self, spider): deferred_list = [] @@ -276,10 +287,17 @@ class FeedExporter: deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None - def _start_new_batch(self, previous_batch_slot, uri, feed, spider): + def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri): """ Redirect the output data stream to a new file. Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero. + :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store + to get the file properly closed. + :param uri: uri of the new batch to start + :param feed: dict with parameters of feed + :param spider: user spider + :param batch_id: sequential batch id starting at 1 + :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri """ if previous_batch_slot is not None: previous_batch_slot.exporter.finish_exporting() @@ -293,30 +311,30 @@ class FeedExporter: encoding=feed['encoding'], indent=feed['indent'] ) - slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty']) + slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'], batch_id, template_uri) if slot.store_empty: slot.start_exporting() return slot - def _get_uri_of_partial(self, template_uri, feed, spider): - """Get uri for each partial using datetime.now().isoformat()""" - template_uri = (template_uri % self._get_uri_params(spider, feed['uri_params'])) - uri_name = template_uri.split('.')[0] - return '{}.{}.{}'.format(uri_name, datetime.now().isoformat(), feed["format"]) - def item_scraped(self, item, spider): slots = [] for idx, slot in enumerate(self.slots): slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - if self.storage_batch and slot.itemcount % self.storage_batch == 0: - uri = self._get_uri_of_partial(slot.uri, self.feeds[slot.uri], spider) - slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider)) - self.feeds[uri] = self.feeds[slot.uri] - self.feeds.pop(slot.uri) + if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0: + batch_id = slot.batch_id + 1 + uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params']) + uri_params['batch_id'] = batch_id + self.slots.append(self._start_new_batch( + previous_batch_slot=slot, + uri=slot.template_uri % uri_params, + feed=self.feeds[slot.template_uri], + spider=spider, + batch_id=batch_id, + template_uri=slot.template_uri + )) self.slots[idx] = None - self.slots = [slot for slot in self.slots if slot is not None] self.slots.extend(slots) @@ -335,6 +353,17 @@ class FeedExporter: return True logger.error("Unknown feed format: %(format)s", {'format': format}) + def _batch_deliveries_supported(self, uri): + """ + If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s + to distinguish different files of partial output + """ + if not self.storage_batch_size: + return True + if '%(time)s' in uri or '%(batch_id)s' in uri: + return True + logger.error('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + def _storage_supported(self, uri): scheme = urlparse(uri).scheme if scheme in self.storages: @@ -364,7 +393,7 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) - ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') + ts = datetime.utcnow().isoformat().replace(':', '-') params['time'] = ts uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 690e044c5..7f90a2280 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH = 0 +FEED_STORAGE_BATCH_SIZE = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', From d11411b402ae68874c6ccc2883836be0b9cf8326 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Sat, 21 Mar 2020 10:48:13 +0300 Subject: [PATCH 062/568] fix comments --- scrapy/extensions/feedexport.py | 31 ++++++++++++++++++----------- scrapy/settings/default_settings.py | 2 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 38b25bf4a..ab0a0de37 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values - logger = logging.getLogger(__name__) @@ -243,7 +242,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_size = self.settings.getint('FEED_STORAGE_BATCH_SIZE') + self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None) for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -263,7 +262,7 @@ class FeedExporter: feed=feed, spider=spider, batch_id=batch_id, - template_uri=uri + template_uri=uri, )) def close_spider(self, spider): @@ -290,7 +289,7 @@ class FeedExporter: def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri): """ Redirect the output data stream to a new file. - Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero. + Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store to get the file properly closed. :param uri: uri of the new batch to start @@ -309,9 +308,18 @@ class FeedExporter: format=feed['format'], fields_to_export=feed['fields'], encoding=feed['encoding'], - indent=feed['indent'] + indent=feed['indent'], + ) + slot = _FeedSlot( + file=file, + exporter=exporter, + storage=storage, + uri=uri, + format=feed['format'], + store_empty=feed['store_empty'], + batch_id=batch_id, + template_uri=template_uri, ) - slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'], batch_id, template_uri) if slot.store_empty: slot.start_exporting() return slot @@ -326,13 +334,13 @@ class FeedExporter: batch_id = slot.batch_id + 1 uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params']) uri_params['batch_id'] = batch_id - self.slots.append(self._start_new_batch( + slots.append(self._start_new_batch( previous_batch_slot=slot, uri=slot.template_uri % uri_params, feed=self.feeds[slot.template_uri], spider=spider, batch_id=batch_id, - template_uri=slot.template_uri + template_uri=slot.template_uri, )) self.slots[idx] = None self.slots = [slot for slot in self.slots if slot is not None] @@ -358,11 +366,10 @@ class FeedExporter: If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s to distinguish different files of partial output """ - if not self.storage_batch_size: + if self.storage_batch_size is None or '%(time)s' in uri or '%(batch_id)s' in uri: return True - if '%(time)s' in uri or '%(batch_id)s' in uri: - return True - logger.error('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + return False def _storage_supported(self, uri): scheme = urlparse(uri).scheme diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7f90a2280..c3463a505 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_SIZE = 0 +FEED_STORAGE_BATCH_SIZE = None FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', From 39d0d13d3f7bd671d5b29646b209c62e23373fab Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 26 Mar 2020 14:18:35 +0300 Subject: [PATCH 063/568] Add partial deliveries tests --- tests/test_feedexport.py | 195 +++++++++++++++++++++++++++++++-------- 1 file changed, 159 insertions(+), 36 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c5589e52f..1ebe44e12 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -6,6 +6,7 @@ import shutil import string import tempfile import warnings +from abc import ABC, abstractmethod from io import BytesIO from pathlib import Path from string import ascii_letters, digits @@ -21,8 +22,9 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner +from scrapy.exceptions import NotConfigured from scrapy.exporters import CsvItemExporter -from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage, +from scrapy.extensions.feedexport import (BlockingFeedStorage, FeedExporter, FileFeedStorage, FTPFeedStorage, IFeedStorage, S3FeedStorage, StdoutFeedStorage) from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -76,6 +78,7 @@ class FTPFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): name = 'test_spider' + crawler = get_crawler(settings_dict=settings) spider = TestSpider.from_crawler(crawler) return spider @@ -129,6 +132,7 @@ class BlockingFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): name = 'test_spider' + crawler = get_crawler(settings_dict=settings) spider = TestSpider.from_crawler(crawler) return spider @@ -390,23 +394,63 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): pass -class FeedExportTest(unittest.TestCase): +class FeedExportTestBase(ABC, unittest.TestCase): + __test__ = False class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() baz = scrapy.Field() + def _random_temp_filename(self, inter_dir=''): + chars = [random.choice(ascii_letters + digits) for _ in range(15)] + filename = ''.join(chars) + return os.path.join(self.temp_dir, inter_dir, filename) + def setUp(self): self.temp_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.temp_dir, ignore_errors=True) - def _random_temp_filename(self): - chars = [random.choice(ascii_letters + digits) for _ in range(15)] - filename = ''.join(chars) - return os.path.join(self.temp_dir, filename) + @defer.inlineCallbacks + def exported_data(self, items, settings): + """ + Return exported data which a spider yielding ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + for item in items: + yield item + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + + @defer.inlineCallbacks + def exported_no_data(self, settings): + """ + Return exported data which a spider yielding no ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + pass + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + + @abstractmethod + def run_and_export(self, spider_cls, settings): + pass + + +class FeedExportTest(FeedExportTestBase): + __test__ = True @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -417,7 +461,6 @@ class FeedExportTest(unittest.TestCase): urljoin('file:', pathname2url(str(file_path))): feed for file_path, feed in FEEDS.items() } - content = {} try: with MockServer() as s: @@ -435,35 +478,6 @@ class FeedExportTest(unittest.TestCase): defer.returnValue(content) - @defer.inlineCallbacks - def exported_data(self, items, settings): - """ - Return exported data which a spider yielding ``items`` would return. - """ - class TestSpider(scrapy.Spider): - name = 'testspider' - - def parse(self, response): - for item in items: - yield item - - data = yield self.run_and_export(TestSpider, settings) - defer.returnValue(data) - - @defer.inlineCallbacks - def exported_no_data(self, settings): - """ - Return exported data which a spider yielding no ``items`` would return. - """ - class TestSpider(scrapy.Spider): - name = 'testspider' - - def parse(self, response): - pass - - data = yield self.run_and_export(TestSpider, settings) - defer.returnValue(data) - @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} @@ -970,3 +984,112 @@ class FeedExportTest(unittest.TestCase): } data = yield self.exported_no_data(settings) self.assertEqual(data['csv'], b'') + + +class PartialDeliveriesTest(FeedExportTestBase): + __test__ = True + _file_mark = '_%(time)s_#%(batch_id)s' + + @defer.inlineCallbacks + def run_and_export(self, spider_cls, settings): + """ Run spider with specified settings; return exported data. """ + + FEEDS = settings.get('FEEDS') or {} + settings['FEEDS'] = { + urljoin('file:', file_path): feed + for file_path, feed in FEEDS.items() + } + from collections import defaultdict + content = defaultdict(list) + try: + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + spider_cls.start_urls = [s.url('/')] + yield runner.crawl(spider_cls) + + for path, feed in FEEDS.items(): + dir_name = os.path.dirname(path) + for file in sorted(os.listdir(dir_name)): + with open(os.path.join(dir_name, file), 'rb') as f: + data = f.read() + content[feed['format']].append(data) + finally: + pass + defer.returnValue(content) + + @defer.inlineCallbacks + def assertPartialExported(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, + }, + }) + data = yield self.exported_data(items, settings) + data['jl'] = b''.join(data['jl']) + parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()] + + rows = [{k: v for k, v in row.items() if v} for row in rows] + self.assertEqual(rows, parsed) + + @defer.inlineCallbacks + def test_partial_deliveries(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + ] + rows = [ + {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, + {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}, + {'foo': 'bar3', 'baz': 'quux3'} + ] + settings = { + 'FEED_STORAGE_BATCH_SIZE': 1 + } + yield self.assertPartialExported(items, rows, settings=settings) + + def test_wrong_path(self): + settings = { + 'FEEDS': { + self._random_temp_filename(): {'format': 'xml'}, + }, + 'FEED_STORAGE_BATCH_SIZE': 1 + } + crawler = get_crawler(settings_dict=settings) + self.assertRaises(NotConfigured, FeedExporter, crawler) + + @defer.inlineCallbacks + def test_export_no_items_not_store_empty(self): + for fmt in ('json', 'jsonlines', 'xml', 'csv'): + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, + }, + 'FEED_STORAGE_BATCH_SIZE': 1 + } + data = yield self.exported_no_data(settings) + data[fmt] = b''.join(data[fmt]) + self.assertEqual(data[fmt], b'') + + @defer.inlineCallbacks + def test_export_no_items_store_empty(self): + formats = ( + ('json', b'[]'), + ('jsonlines', b''), + ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'), + ('csv', b''), + ) + + for fmt, expctd in formats: + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, + }, + 'FEED_STORE_EMPTY': True, + 'FEED_EXPORT_INDENT': None, + 'FEED_STORAGE_BATCH_SIZE': 1 + } + data = yield self.exported_no_data(settings) + data[fmt] = b''.join(data[fmt]) + self.assertEqual(data[fmt], expctd) From ffa8a533e74478a5c81fbf453f2c65601bb1d244 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Sat, 28 Mar 2020 11:40:16 +0300 Subject: [PATCH 064/568] Set batch_id in _get_uri_params --- scrapy/extensions/feedexport.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index ab0a0de37..06ea6c5b2 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -253,15 +253,12 @@ class FeedExporter: def open_spider(self, spider): for uri, feed in self.feeds.items(): - batch_id = 1 - uri_params = self._get_uri_params(spider, feed['uri_params']) - uri_params['batch_id'] = batch_id + uri_params = self._get_uri_params(spider, feed['uri_params'], None) self.slots.append(self._start_new_batch( previous_batch_slot=None, uri=uri % uri_params, feed=feed, spider=spider, - batch_id=batch_id, template_uri=uri, )) @@ -286,7 +283,7 @@ class FeedExporter: deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None - def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri): + def _start_new_batch(self, previous_batch_slot, uri, feed, spider, template_uri): """ Redirect the output data stream to a new file. Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. @@ -295,12 +292,15 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param batch_id: sequential batch id starting at 1 :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri """ if previous_batch_slot is not None: + previous_batch_id = previous_batch_slot.batch_id previous_batch_slot.exporter.finish_exporting() previous_batch_slot.storage.store(previous_batch_slot.file) + else: + previous_batch_id = 0 + storage = self._get_storage(uri) file = storage.open(spider) exporter = self._get_exporter( @@ -317,7 +317,7 @@ class FeedExporter: uri=uri, format=feed['format'], store_empty=feed['store_empty'], - batch_id=batch_id, + batch_id=previous_batch_id + 1, template_uri=template_uri, ) if slot.store_empty: @@ -331,15 +331,12 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0: - batch_id = slot.batch_id + 1 - uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params']) - uri_params['batch_id'] = batch_id + uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) slots.append(self._start_new_batch( previous_batch_slot=slot, uri=slot.template_uri % uri_params, feed=self.feeds[slot.template_uri], spider=spider, - batch_id=batch_id, template_uri=slot.template_uri, )) self.slots[idx] = None @@ -396,12 +393,12 @@ class FeedExporter: def _get_storage(self, uri): return self._get_instance(self.storages[urlparse(uri).scheme], uri) - def _get_uri_params(self, spider, uri_params): + def _get_uri_params(self, spider, uri_params, slot): params = {} for k in dir(spider): params[k] = getattr(spider, k) - ts = datetime.utcnow().isoformat().replace(':', '-') - params['time'] = ts + params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 + params['time'] = datetime.utcnow().isoformat().replace(':', '-') uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) return params From 963580463b96315eb58319e6d35b4cd52672371a Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Wed, 15 Apr 2020 20:14:33 +0300 Subject: [PATCH 065/568] Update tests --- tests/test_feedexport.py | 203 ++++++++++++++++++++++++++++++--------- 1 file changed, 159 insertions(+), 44 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1ebe44e12..c6cd867b1 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -7,6 +7,7 @@ import string import tempfile import warnings from abc import ABC, abstractmethod +from collections import defaultdict from io import BytesIO from pathlib import Path from string import ascii_letters, digits @@ -444,10 +445,31 @@ class FeedExportTestBase(ABC, unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) defer.returnValue(data) + @defer.inlineCallbacks + def assertExported(self, items, header, rows, settings=None, ordered=True): + yield self.assertExportedCsv(items, header, rows, settings, ordered) + yield self.assertExportedJsonLines(items, rows, settings) + yield self.assertExportedXml(items, rows, settings) + yield self.assertExportedPickle(items, rows, settings) + yield self.assertExportedMarshal(items, rows, settings) + yield self.assertExportedMultiple(items, rows, settings) + @abstractmethod def run_and_export(self, spider_cls, settings): pass + def _load_until_eof(self, data, load_func): + result = [] + with tempfile.TemporaryFile() as temp: + temp.write(data) + temp.seek(0) + while True: + try: + result.append(load_func(temp)) + except EOFError: + break + return result + class FeedExportTest(FeedExportTestBase): __test__ = True @@ -478,6 +500,22 @@ class FeedExportTest(FeedExportTestBase): defer.returnValue(content) + @defer.inlineCallbacks + def exported_data(self, items, settings): + """ + Return exported data which a spider yielding ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + for item in items: + yield item + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} @@ -543,18 +581,6 @@ class FeedExportTest(FeedExportTestBase): json_rows = json.loads(to_unicode(data['json'])) self.assertEqual(rows, json_rows) - def _load_until_eof(self, data, load_func): - result = [] - with tempfile.TemporaryFile() as temp: - temp.write(data) - temp.seek(0) - while True: - try: - result.append(load_func(temp)) - except EOFError: - break - return result - @defer.inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} @@ -583,15 +609,6 @@ class FeedExportTest(FeedExportTestBase): result = self._load_until_eof(data['marshal'], load_func=marshal.load) self.assertEqual(expected, result) - @defer.inlineCallbacks - def assertExported(self, items, header, rows, settings=None, ordered=True): - yield self.assertExportedCsv(items, header, rows, settings, ordered) - yield self.assertExportedJsonLines(items, rows, settings) - yield self.assertExportedXml(items, rows, settings) - yield self.assertExportedPickle(items, rows, settings) - yield self.assertExportedMarshal(items, rows, settings) - yield self.assertExportedMultiple(items, rows, settings) - @defer.inlineCallbacks def test_export_items(self): # feed exporters use field names from Item @@ -615,7 +632,7 @@ class FeedExportTest(FeedExportTestBase): }, } data = yield self.exported_no_data(settings) - self.assertEqual(data[fmt], b'') + self.assertEqual(b'', data[fmt]) @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -635,7 +652,7 @@ class FeedExportTest(FeedExportTestBase): 'FEED_EXPORT_INDENT': None, } data = yield self.exported_no_data(settings) - self.assertEqual(data[fmt], expctd) + self.assertEqual(expctd, data[fmt]) @defer.inlineCallbacks def test_export_multiple_item_classes(self): @@ -734,7 +751,8 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), - 'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'), + 'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode( + 'utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } @@ -751,7 +769,8 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), - 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'), + 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode( + 'latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } @@ -772,7 +791,8 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), - 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode('latin-1'), + 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode( + 'latin-1'), 'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), } @@ -988,7 +1008,7 @@ class FeedExportTest(FeedExportTestBase): class PartialDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(time)s_#%(batch_id)s' + _file_mark = '_%(time)s_#%(batch_id)s_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -999,7 +1019,6 @@ class PartialDeliveriesTest(FeedExportTestBase): urljoin('file:', file_path): feed for file_path, feed in FEEDS.items() } - from collections import defaultdict content = defaultdict(list) try: with MockServer() as s: @@ -1014,26 +1033,120 @@ class PartialDeliveriesTest(FeedExportTestBase): data = f.read() content[feed['format']].append(data) finally: - pass + self.tearDown() defer.returnValue(content) @defer.inlineCallbacks - def assertPartialExported(self, items, rows, settings=None): + def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': { os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - data = yield self.exported_data(items, settings) - data['jl'] = b''.join(data['jl']) - parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()] - + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] rows = [{k: v for k, v in row.items() if v} for row in rows] - self.assertEqual(rows, parsed) + data = yield self.exported_data(items, settings) + for batch in data['jl']: + got_batch = [json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks - def test_partial_deliveries(self): + def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + data = yield self.exported_data(items, settings) + for batch in data['csv']: + got_batch = csv.DictReader(to_unicode(batch).splitlines()) + self.assertEqual(list(header), got_batch.fieldnames) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, list(got_batch)) + + @defer.inlineCallbacks + def assertExportedXml(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + for batch in data['xml']: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedMultiple(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, + os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + # XML + xml_rows = rows.copy() + for batch in data['xml']: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')] + expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + # JSON + json_rows = rows.copy() + for batch in data['json']: + got_batch = json.loads(batch) + expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedPickle(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + import pickle + for batch in data['pickle']: + got_batch = self._load_until_eof(batch, load_func=pickle.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedMarshal(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + import marshal + for batch in data['marshal']: + got_batch = self._load_until_eof(batch, load_func=marshal.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def test_export_items(self): items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), @@ -1042,14 +1155,16 @@ class PartialDeliveriesTest(FeedExportTestBase): rows = [ {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}, - {'foo': 'bar3', 'baz': 'quux3'} + {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} ] settings = { - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_SIZE': 2 } - yield self.assertPartialExported(items, rows, settings=settings) + header = self.MyItem.fields.keys() + yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): + """If path without %(time)s or %(batch_id)s an exception must be raised""" settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1069,8 +1184,8 @@ class PartialDeliveriesTest(FeedExportTestBase): 'FEED_STORAGE_BATCH_SIZE': 1 } data = yield self.exported_no_data(settings) - data[fmt] = b''.join(data[fmt]) - self.assertEqual(data[fmt], b'') + data = dict(data) + self.assertEqual(b'', data[fmt][0]) @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -1088,8 +1203,8 @@ class PartialDeliveriesTest(FeedExportTestBase): }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_SIZE': 1, } data = yield self.exported_no_data(settings) - data[fmt] = b''.join(data[fmt]) - self.assertEqual(data[fmt], expctd) + data = dict(data) + self.assertEqual(expctd, data[fmt][0]) From cac1f3a6adedc32977e0fb1830917a5e7d758bef Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 16 Apr 2020 10:06:56 +0300 Subject: [PATCH 066/568] Update documentation --- docs/topics/feed-exports.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 9e5968a29..0bba03a7c 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -220,6 +220,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` + * :setting:`FEED_EXPORT_BATCH_SIZE` .. currentmodule:: scrapy.extensions.feedexport @@ -429,3 +430,37 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _Amazon S3: https://aws.amazon.com/s3/ .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + +.. setting:: FEED_EXPORT_BATCH_SIZE + +FEED_EXPORT_BATCH_SIZE +---------------------- +Default: ``None`` + +An integer number which represent number of scraped items stored in each output +file. Whenever the number of items exceeds this setting, a new file +creates and output redirects to it. The name of the new file will be selected +based on timestamp when the feed is being created and/or batch sequence number. +Therefore you must specify %(time)s or %(batch_id)s or both in the file path. + +* ``%(time)s`` - gets replaced by a timestamp when the feed is being created +* ``%(batch_id)s`` - gets replaced by sequence number of batch + +For instance:: + + FEED_EXPORT_BATCH_SIZE=100 + +Your request can be like:: + + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time)s.json + +The result directory tree of above can be like:: + +->projectname +-->dirname +--->1-filename2020-03-28T14-45-08.237134.json +--->2-filename2020-03-28T14-45-09.148903.json +--->3-filename2020-03-28T14-45-10.046092.json + +Where first and second files contain exactly 100 items. The last one contains +<= 100 items. \ No newline at end of file From 5980ae72c6cb177f47fbb41d17837e8d98d50025 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 16 Apr 2020 10:13:39 +0300 Subject: [PATCH 067/568] Some minor fixes and refactoring --- tests/test_feedexport.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 60e19d1df..e97e50e8e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -428,7 +428,7 @@ class FeedExportTestBase(ABC, unittest.TestCase): yield item data = yield self.run_and_export(TestSpider, settings) - defer.returnValue(data) + return data @defer.inlineCallbacks def exported_no_data(self, settings): @@ -443,7 +443,7 @@ class FeedExportTestBase(ABC, unittest.TestCase): pass data = yield self.run_and_export(TestSpider, settings) - defer.returnValue(data) + return data @defer.inlineCallbacks def assertExported(self, items, header, rows, settings=None, ordered=True): @@ -735,8 +735,7 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), - 'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode( - 'utf-8'), + 'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } @@ -753,8 +752,7 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), - 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode( - 'latin-1'), + 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } @@ -775,8 +773,7 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), - 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode( - 'latin-1'), + 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode('latin-1'), 'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), } @@ -1148,7 +1145,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """If path without %(time)s or %(batch_id)s an exception must be raised""" + """If path is without %(time)s or %(batch_id)s an exception must be raised""" settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, From 1d77eac950966463faee411b6196f197a8d32d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 16 Apr 2020 14:57:55 +0200 Subject: [PATCH 068/568] Fix Flake8-reported issues --- scrapy/http/request/form.py | 2 +- tests/test_http_request.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d58d217b6..bdb6bec7a 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -33,7 +33,7 @@ class FormRequest(Request): self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') self._set_body(form_query_str) else: - self._set_url(urlunsplit(urlsplit(self.url)._replace(query = form_query_str))) + self._set_url(urlunsplit(urlsplit(self.url)._replace(query=form_query_str))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, diff --git a/tests/test_http_request.py b/tests/test_http_request.py index bc34bb26d..96954d419 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -369,7 +369,7 @@ class FormRequestTest(RequestTest): def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) self.assertEqual(r1.body, b'') - + def test_formdata_overrides_querystring(self): data = (('a', 'one'), ('a', 'two'), ('b', '2')) url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] @@ -378,7 +378,7 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'b'], [b'2']) self.assertIsNone(fs.get(b'c')) - data = {'a' : '1', 'b' : '2'} + data = {'a': '1', 'b': '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) From 94ee68695a42ec8d3ccdff71bfc2fa33d5ea7049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 16 Apr 2020 15:36:56 +0200 Subject: [PATCH 069/568] Mock server: use 127.0.0.1 also for HTTPS Windows throws an error about 0.0.0.0 being external: https://stackoverflow.com/a/23857995/939364 --- tests/mockserver.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index a45277db9..e3dbdcc68 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -218,9 +218,8 @@ class MockServer(): self.proc.communicate() def url(self, path, is_secure=False): - host = self.http_address.replace('0.0.0.0', '127.0.0.1') - if is_secure: - host = self.https_address + host = self.https_address if is_secure else self.http_address + host = host.replace('0.0.0.0', '127.0.0.1') return host + path From 7cc9601029274124804e63ebebfa8783ac175205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 16 Apr 2020 16:57:48 +0200 Subject: [PATCH 070/568] Improve reporting on test_ipv6_alternative_name_resolver --- tests/test_crawler.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 169e763f0..a6b079395 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -309,14 +309,8 @@ class CrawlerProcessSubprocess(unittest.TestCase): def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertTrue(any([ - "twisted.internet.error.ConnectionRefusedError" in log, - "twisted.internet.error.ConnectError" in log, - ])) - self.assertTrue(any([ - "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log, - "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, - ])) + self.assertRegex(log, r"twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)") + self.assertRegex(log, r"'downloader/exception_type_count/twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)': 1,") def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") From cf4180308982d9cd017167238a3df8de7900646d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 16 Apr 2020 17:07:29 +0200 Subject: [PATCH 071/568] Skip test_reactor_poll on Windows --- tests/test_crawler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index a6b079395..3d166e14c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,6 @@ import logging import os +import platform import subprocess import sys import warnings @@ -317,6 +318,7 @@ class CrawlerProcessSubprocess(unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + @mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows") def test_reactor_poll(self): log = self.run_script("twisted_reactor_poll.py") self.assertIn("Spider closed (finished)", log) From ea3e675801fe41c7517c58e21b46831940fbd064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 16 Apr 2020 17:10:45 +0200 Subject: [PATCH 072/568] test_utils_iterators: use os.linesep --- tests/test_utils_iterators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 33fc4d570..5b0073fd1 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -8,7 +8,7 @@ from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata -FOOBAR_NL = u"foo\nbar" +FOOBAR_NL = "foo{}bar".format(os.linesep) class XmliterTestCase(unittest.TestCase): From ec76445dd6753074c1531571f66467eecf22b498 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Sat, 18 Apr 2020 09:29:23 +0300 Subject: [PATCH 073/568] Update tests --- tests/test_feedexport.py | 66 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e97e50e8e..8e03a91c8 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1128,6 +1128,7 @@ class PartialDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_items(self): + """ Test partial deliveries in all supported formats """ items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), @@ -1145,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """If path is without %(time)s or %(batch_id)s an exception must be raised""" + """ If path is without %(time)s or %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1189,3 +1190,66 @@ class PartialDeliveriesTest(FeedExportTestBase): data = yield self.exported_no_data(settings) data = dict(data) self.assertEqual(expctd, data[fmt][0]) + + @defer.inlineCallbacks + def test_export_multiple_configs(self): + items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] + + formats = { + 'json': [u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), + u'[\n{"bar": "BAR1"}\n]'.encode('utf-8')], + 'xml': [u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode('latin-1'), + u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>'.encode('latin-1')], + 'csv': [u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), + u'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')], + } + + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + 'format': 'json', + 'indent': 0, + 'fields': ['bar'], + 'encoding': 'utf-8', + }, + os.path.join(self._random_temp_filename(), 'xml', self._file_mark): { + 'format': 'xml', + 'indent': 2, + 'fields': ['foo'], + 'encoding': 'latin-1', + }, + os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { + 'format': 'csv', + 'indent': None, + 'fields': ['bar', 'foo'], + 'encoding': 'utf-8', + }, + }, + 'FEED_STORAGE_BATCH_SIZE': 1, + } + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt]): + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def test_batch_path_differ(self): + """ + Test that the name of all batch files differ from each other. + So %(time)s replaced with the current date. + """ + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + ] + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), '%(time)s'): { + 'format': 'json', + }, + }, + 'FEED_STORAGE_BATCH_SIZE': 1, + } + data = yield self.exported_data(items, settings) + self.assertEqual(len(items) + 1, len(data['json'])) From 1fecacbb1a3c2c4c61c202561f9c51f7f6b191ba Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Mon, 20 Apr 2020 12:05:15 -0300 Subject: [PATCH 074/568] IPv6 test: check for the absence of DNSLookupError --- tests/test_crawler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 3d166e14c..d6756c266 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -310,8 +310,7 @@ class CrawlerProcessSubprocess(unittest.TestCase): def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertRegex(log, r"twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)") - self.assertRegex(log, r"'downloader/exception_type_count/twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)': 1,") + self.assertNotIn("twisted.internet.error.DNSLookupError", log) def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") From f0f1be76d1e6cef65ac9a01d13c5d5060a03f648 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Mon, 27 Apr 2020 09:56:57 +0300 Subject: [PATCH 075/568] Using time_id instead of time as a timestamp --- docs/topics/feed-exports.rst | 6 +++--- scrapy/extensions/feedexport.py | 11 ++++++----- tests/test_feedexport.py | 8 ++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0bba03a7c..2017be78f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output file. Whenever the number of items exceeds this setting, a new file creates and output redirects to it. The name of the new file will be selected based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(time)s or %(batch_id)s or both in the file path. +Therefore you must specify %(time_id)s or %(batch_id)s or both in the file path. -* ``%(time)s`` - gets replaced by a timestamp when the feed is being created +* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created * ``%(batch_id)s`` - gets replaced by sequence number of batch For instance:: @@ -452,7 +452,7 @@ For instance:: Your request can be like:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time)s.json + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json The result directory tree of above can be like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 06ea6c5b2..72baa6269 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -292,7 +292,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri + :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri """ if previous_batch_slot is not None: previous_batch_id = previous_batch_slot.batch_id @@ -360,12 +360,12 @@ class FeedExporter: def _batch_deliveries_supported(self, uri): """ - If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s + If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s to distinguish different files of partial output """ - if self.storage_batch_size is None or '%(time)s' in uri or '%(batch_id)s' in uri: + if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') return False def _storage_supported(self, uri): @@ -397,8 +397,9 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) + params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') + params['time_id'] = datetime.utcnow().isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 - params['time'] = datetime.utcnow().isoformat().replace(':', '-') uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) return params diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8e03a91c8..da759917a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase): class PartialDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(time)s_#%(batch_id)s_' + _file_mark = '_%(time_id)s_#%(batch_id)s_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """ If path is without %(time)s or %(batch_id)s an exception must be raised """ + """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase): def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. - So %(time)s replaced with the current date. + So %(time_id)s replaced with the current date. """ items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase): ] settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), '%(time)s'): { + os.path.join(self._random_temp_filename(), '%(time_id)s'): { 'format': 'json', }, }, From 2eee6c81017e08bb492da560bc73c03f4f375fcc Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Mon, 27 Apr 2020 09:58:14 +0300 Subject: [PATCH 076/568] Documentation spelling fix --- docs/topics/feed-exports.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2017be78f..6c463fc27 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -438,10 +438,10 @@ FEED_EXPORT_BATCH_SIZE Default: ``None`` An integer number which represent number of scraped items stored in each output -file. Whenever the number of items exceeds this setting, a new file -creates and output redirects to it. The name of the new file will be selected +file. Whenever the number of items exceeds this setting, a new file is +created and output redirects to it. The name of the new file will be selected based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(time_id)s or %(batch_id)s or both in the file path. +Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI. * ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created * ``%(batch_id)s`` - gets replaced by sequence number of batch From 204737042ac6672eee73c975d0bd6735893d684c Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Mon, 27 Apr 2020 12:52:18 +0300 Subject: [PATCH 077/568] Extract the slot closing functionality to the function; minor changes --- scrapy/extensions/feedexport.py | 56 ++++++++++++++++----------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 72baa6269..fe6061c33 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -255,7 +255,7 @@ class FeedExporter: for uri, feed in self.feeds.items(): uri_params = self._get_uri_params(spider, feed['uri_params'], None) self.slots.append(self._start_new_batch( - previous_batch_slot=None, + batch_id=1, uri=uri % uri_params, feed=feed, spider=spider, @@ -265,42 +265,38 @@ class FeedExporter: def close_spider(self, spider): deferred_list = [] for slot in self.slots: - if not slot.itemcount and not slot.store_empty: - # We need to call slot.storage.store nonetheless to get the file - # properly closed. - return defer.maybeDeferred(slot.storage.store, slot.file) - slot.finish_exporting() - logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" - log_args = {'format': slot.format, - 'itemcount': slot.itemcount, - 'uri': slot.uri} - d = defer.maybeDeferred(slot.storage.store, slot.file) - d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args, - extra={'spider': spider})) - d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args, - exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + d = self._close_slot(slot, spider) deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None - def _start_new_batch(self, previous_batch_slot, uri, feed, spider, template_uri): + def _close_slot(self, slot, spider): + if not slot.itemcount and not slot.store_empty: + # We need to call slot.storage.store nonetheless to get the file + # properly closed. + return defer.maybeDeferred(slot.storage.store, slot.file) + slot.finish_exporting() + logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" + log_args = {'format': slot.format, + 'itemcount': slot.itemcount, + 'uri': slot.uri} + d = defer.maybeDeferred(slot.storage.store, slot.file) + d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args, + extra={'spider': spider})) + d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args, + exc_info=failure_to_exc_info(f), + extra={'spider': spider})) + return d + + def _start_new_batch(self, batch_id, uri, feed, spider, template_uri): """ Redirect the output data stream to a new file. Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. - :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store - to get the file properly closed. + :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri """ - if previous_batch_slot is not None: - previous_batch_id = previous_batch_slot.batch_id - previous_batch_slot.exporter.finish_exporting() - previous_batch_slot.storage.store(previous_batch_slot.file) - else: - previous_batch_id = 0 - storage = self._get_storage(uri) file = storage.open(spider) exporter = self._get_exporter( @@ -317,7 +313,7 @@ class FeedExporter: uri=uri, format=feed['format'], store_empty=feed['store_empty'], - batch_id=previous_batch_id + 1, + batch_id=batch_id, template_uri=template_uri, ) if slot.store_empty: @@ -330,10 +326,12 @@ class FeedExporter: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0: + # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one + if self.storage_batch_size and slot.itemcount == self.storage_batch_size: uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) + self._close_slot(slot, spider) slots.append(self._start_new_batch( - previous_batch_slot=slot, + batch_id=slot.batch_id + 1, uri=slot.template_uri % uri_params, feed=self.feeds[slot.template_uri], spider=spider, From 3f9874fac9f93c0956afa5975d7b2bbb21816894 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Fri, 1 May 2020 11:52:16 +0300 Subject: [PATCH 078/568] Add test s3 export --- tests/test_feedexport.py | 70 ++++++++++++++++++++++++++++++++++++++++ tox.ini | 1 + 2 files changed, 71 insertions(+) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index da759917a..9fc39c3a6 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1253,3 +1253,73 @@ class PartialDeliveriesTest(FeedExportTestBase): } data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) + + @defer.inlineCallbacks + def test_s3_export(self): + """ + Test export of items into s3 bucket. + S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini + to perform this test: + [testenv] + setenv = + AWS_SECRET_ACCESS_KEY = ABCD + AWS_ACCESS_KEY_ID = ABCD + S3_TEST_BUCKET_NAME = ABCD + """ + try: + import boto3 + except ImportError: + raise unittest.SkipTest("S3FeedStorage requires boto3") + + assert_aws_environ() + s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME') + access_key = os.environ.get('AWS_ACCESS_KEY_ID') + secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') + if not s3_test_bucket_name: + raise unittest.SkipTest("No S3 BUCKET available for testing") + + chars = [random.choice(ascii_letters + digits) for _ in range(15)] + filename = ''.join(chars) + prefix = 'tmp/{filename}'.format(filename=filename) + s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format( + bucket_name=s3_test_bucket_name, prefix=prefix + ) + storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) + settings = { + 'FEEDS': { + s3_test_file_uri: { + 'format': 'json', + }, + }, + 'FEED_STORAGE_BATCH_SIZE': 1, + } + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + ] + verifyObject(IFeedStorage, storage) + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + for item in items: + yield item + + s3 = boto3.resource('s3') + my_bucket = s3.Bucket(s3_test_bucket_name) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + TestSpider.start_urls = [s.url('/')] + yield runner.crawl(TestSpider) + + for file_uri in my_bucket.objects.filter(Prefix=prefix): + content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key) + if not content and not items: + break + content = json.loads(content.decode('utf-8')) + expected_batch, items = items[:batch_size], items[batch_size:] + self.assertEqual(expected_batch, content) diff --git a/tox.ini b/tox.ini index cd118c921..c77fae1f0 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ deps = # Extras botocore>=1.3.23 Pillow>=3.4.2 + boto3>=1.13.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From dad2ea75222d6240c569440d3221f5fc00925682 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Sat, 2 May 2020 01:21:03 +0300 Subject: [PATCH 079/568] Change time_id to batch_time --- docs/topics/feed-exports.rst | 6 +++--- scrapy/extensions/feedexport.py | 10 +++++----- tests/test_feedexport.py | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 6c463fc27..2106b41f5 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output file. Whenever the number of items exceeds this setting, a new file is created and output redirects to it. The name of the new file will be selected based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI. +Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI. -* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created +* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created * ``%(batch_id)s`` - gets replaced by sequence number of batch For instance:: @@ -452,7 +452,7 @@ For instance:: Your request can be like:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json The result directory tree of above can be like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fe6061c33..a262f5d18 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -295,7 +295,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri + :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri """ storage = self._get_storage(uri) file = storage.open(spider) @@ -358,12 +358,12 @@ class FeedExporter: def _batch_deliveries_supported(self, uri): """ - If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s + If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ - if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri: + if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') return False def _storage_supported(self, uri): @@ -396,7 +396,7 @@ class FeedExporter: for k in dir(spider): params[k] = getattr(spider, k) params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') - params['time_id'] = datetime.utcnow().isoformat().replace(':', '-') + params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 9fc39c3a6..2217bb4ed 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase): class PartialDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(time_id)s_#%(batch_id)s_' + _file_mark = '_%(batch_time)s_#%(batch_id)s_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """ + """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase): def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. - So %(time_id)s replaced with the current date. + So %(batch_time)s replaced with the current date. """ items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase): ] settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), '%(time_id)s'): { + os.path.join(self._random_temp_filename(), '%(batch_time)s'): { 'format': 'json', }, }, @@ -1281,7 +1281,7 @@ class PartialDeliveriesTest(FeedExportTestBase): chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) prefix = 'tmp/{filename}'.format(filename=filename) - s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format( + s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format( bucket_name=s3_test_bucket_name, prefix=prefix ) storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) From 286fca733f23fa41165edcdcc7ab7593cc6b074f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 6 May 2020 16:20:33 +0200 Subject: [PATCH 080/568] Fix parameter name, broken by copy-pasting --- scrapy/settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 8f6fd3e6a..99ffa0dc9 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -248,7 +248,7 @@ class BaseSettings(MutableMapping): :type name: str :param value: the value to associate with the setting - :type default: object + :type value: object :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer From bbd9d05dbff7abb6ab3a5ea575d75cfe88cb2ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 7 May 2020 11:44:43 +0200 Subject: [PATCH 081/568] request-response.rst: review type references around body mentions --- docs/topics/request-response.rst | 38 +++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1638c202d..4fec70e13 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -51,12 +51,10 @@ Request objects given, the dict passed in this parameter will be shallow copied. :type meta: dict - :param body: the request body. If a ``unicode`` is passed, then it's encoded to - ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If - ``body`` is not given, an empty string is stored. Regardless of the - type of this argument, the final value stored will be a ``str`` (never - ``unicode`` or ``None``). - :type body: str + :param body: the request body. If a string is passed, it is converted to + bytes using *encoding*, which defaults to ``utf-8``. If not passed or + ``None`` is passed, an empty bytes array is stored. + :type body: bytes :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If @@ -106,7 +104,7 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the - body to ``str`` (if given as ``unicode``). + body to bytes if given as a string. :type encoding: str :param priority: the priority of this request (defaults to ``0``). @@ -159,7 +157,7 @@ Request objects .. attribute:: Request.body - A str that contains the request body. + The request body as bytes. This attribute is read-only. To change the body of a Request use :meth:`replace`. @@ -598,7 +596,7 @@ Response objects (for single valued headers) or lists (for multi-valued headers). :type headers: dict - :param body: the response body. To access the decoded text as str you can use + :param body: the response body. To access the decoded text as a string, use ``response.text`` from an encoding-aware :ref:`Response subclass <topics-request-response-ref-response-subclasses>`, such as :class:`TextResponse`. @@ -646,10 +644,10 @@ Response objects .. attribute:: Response.body - The body of this Response. Keep in mind that Response.body - is always a bytes object. If you want the unicode version use - :attr:`TextResponse.text` (only available in :class:`TextResponse` - and subclasses). + The response body as bytes. + + If you want the body as a string, use :attr:`TextResponse.text` (only + available in :class:`TextResponse` and subclasses). This attribute is read-only. To change the body of a Response use :meth:`replace`. @@ -768,10 +766,10 @@ TextResponse objects is the same as for the :class:`Response` class and is not documented here. :param encoding: is a string which contains the encoding to use for this - response. If you create a :class:`TextResponse` object with a unicode - body, it will be encoded using this encoding (remember the body attribute - is always a string). If ``encoding`` is ``None`` (default value), the - encoding will be looked up in the response headers and body instead. + response. If you create a :class:`TextResponse` object with a string as + body, it will be converted to bytes encoded using this encoding. If + *encoding* is ``None`` (default), the encoding will be looked up in the + response headers and body instead. :type encoding: str :class:`TextResponse` objects support the following attributes in addition @@ -779,7 +777,7 @@ TextResponse objects .. attribute:: TextResponse.text - Response body, as unicode. + Response body as a string. The same as ``response.body.decode(response.encoding)``, but the result is cached after the first call, so you can access @@ -787,8 +785,8 @@ TextResponse objects .. note:: - ``unicode(response.body)`` is not a correct way to convert response - body to unicode: you would be using the system default encoding + ``str(response.body)`` is not a correct way to convert the response + body into a string: you would be using the system default encoding (typically ``ascii``) instead of the response encoding. From c6746f0e381a44f3d66efad86a4aca87805138bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Sat, 9 May 2020 15:51:11 +0200 Subject: [PATCH 082/568] =?UTF-8?q?bytes=20array=20=E2=86=92=20bytes=20obj?= =?UTF-8?q?ect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 4fec70e13..ad6c10b6e 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -53,7 +53,7 @@ Request objects :param body: the request body. If a string is passed, it is converted to bytes using *encoding*, which defaults to ``utf-8``. If not passed or - ``None`` is passed, an empty bytes array is stored. + ``None`` is passed, an empty :class:`bytes` object is stored. :type body: bytes :param headers: the headers of this request. The dict values can be strings From e07708e3744fb26fa72042720b33deab59331cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Sat, 9 May 2020 15:54:31 +0200 Subject: [PATCH 083/568] request-response: update the consequences of str(b'') --- docs/topics/request-response.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ad6c10b6e..397632932 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -786,8 +786,7 @@ TextResponse objects .. note:: ``str(response.body)`` is not a correct way to convert the response - body into a string: you would be using the system default encoding - (typically ``ascii``) instead of the response encoding. + body into a string: ``str(b'')`` returns ``"b''"``. .. attribute:: TextResponse.encoding From b5684909d1cb01ad138a389caa750485b51f79cf Mon Sep 17 00:00:00 2001 From: Jacty <jacty@users.noreply.github.com> Date: Mon, 11 May 2020 11:18:25 +0800 Subject: [PATCH 084/568] Unnecessary update when value is None When value is None, it is not necessary to invoke update and run other methods and conditions to make the code complicated there. --- scrapy/settings/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b9a13c018..f28fbfaf9 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -83,7 +83,8 @@ class BaseSettings(MutableMapping): def __init__(self, values=None, priority='project'): self.frozen = False self.attributes = {} - self.update(values, priority) + if values is not None: + self.update(values, priority) def __getitem__(self, opt_name): if opt_name not in self: From 33ab0a36635fbd45debbc44584002bd7a4ef7fed Mon Sep 17 00:00:00 2001 From: Jacty <jacty@users.noreply.github.com> Date: Wed, 13 May 2020 06:11:07 +0800 Subject: [PATCH 085/568] Update __init__.py --- scrapy/settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index f28fbfaf9..0425b48b3 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -83,7 +83,7 @@ class BaseSettings(MutableMapping): def __init__(self, values=None, priority='project'): self.frozen = False self.attributes = {} - if values is not None: + if values: self.update(values, priority) def __getitem__(self, opt_name): From 2327ecead085a41d1a71a70a12eb988bbf982268 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Wed, 13 May 2020 22:50:04 +0300 Subject: [PATCH 086/568] Rename FEED_STORAGE_BATCH_SIZE to FEED_STORAGE_BATCH_ITEM_COUNT --- docs/topics/feed-exports.rst | 8 ++++---- scrapy/extensions/feedexport.py | 10 +++++----- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 28 ++++++++++++++-------------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2106b41f5..917240d4d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` - * :setting:`FEED_EXPORT_BATCH_SIZE` + * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport @@ -431,9 +431,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl -.. setting:: FEED_EXPORT_BATCH_SIZE +.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT -FEED_EXPORT_BATCH_SIZE +FEED_STORAGE_BATCH_ITEM_COUNT ---------------------- Default: ``None`` @@ -448,7 +448,7 @@ Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI. For instance:: - FEED_EXPORT_BATCH_SIZE=100 + FEED_STORAGE_BATCH_ITEM_COUNT=100 Your request can be like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index a262f5d18..5bc946634 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -242,7 +242,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None) + self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -290,7 +290,7 @@ class FeedExporter: def _start_new_batch(self, batch_id, uri, feed, spider, template_uri): """ Redirect the output data stream to a new file. - Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. + Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified. :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed @@ -326,7 +326,7 @@ class FeedExporter: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one + # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one if self.storage_batch_size and slot.itemcount == self.storage_batch_size: uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) self._close_slot(slot, spider) @@ -358,12 +358,12 @@ class FeedExporter: def _batch_deliveries_supported(self, uri): """ - If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s + If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') return False def _storage_supported(self, uri): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c3463a505..5a7dc533e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_SIZE = None +FEED_STORAGE_BATCH_ITEM_COUNT = None FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2217bb4ed..1a21eeba9 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1025,7 +1025,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -1041,7 +1041,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -1057,7 +1057,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -1075,7 +1075,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -1100,7 +1100,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -1117,7 +1117,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -1140,7 +1140,7 @@ class PartialDeliveriesTest(FeedExportTestBase): {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} ] settings = { - 'FEED_STORAGE_BATCH_SIZE': 2 + 'FEED_STORAGE_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, settings=settings) @@ -1151,7 +1151,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, }, - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 } crawler = get_crawler(settings_dict=settings) self.assertRaises(NotConfigured, FeedExporter, crawler) @@ -1163,7 +1163,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'FEEDS': { os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, }, - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 } data = yield self.exported_no_data(settings) data = dict(data) @@ -1185,7 +1185,7 @@ class PartialDeliveriesTest(FeedExportTestBase): }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } data = yield self.exported_no_data(settings) data = dict(data) @@ -1225,7 +1225,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'encoding': 'utf-8', }, }, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): @@ -1249,7 +1249,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) @@ -1291,7 +1291,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1309,7 +1309,7 @@ class PartialDeliveriesTest(FeedExportTestBase): s3 = boto3.resource('s3') my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] with MockServer() as s: runner = CrawlerRunner(Settings(settings)) From 8662d3587df74841d4ea640c0432446569e59262 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Wed, 13 May 2020 23:41:01 +0300 Subject: [PATCH 087/568] Documentation and code refactoring --- docs/topics/feed-exports.rst | 21 ++++++++++++--------- scrapy/extensions/feedexport.py | 7 ++++--- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 917240d4d..0f15044b3 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -437,24 +437,27 @@ FEED_STORAGE_BATCH_ITEM_COUNT ---------------------- Default: ``None`` -An integer number which represent number of scraped items stored in each output +An integer number that represents the number of scraped items stored in each output file. Whenever the number of items exceeds this setting, a new file is -created and output redirects to it. The name of the new file will be selected -based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI. +created and the output is redirected to it. The name of the new file will be selected +based on the timestamp when the feed is being created and/or on the batch sequence number. +Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`. * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created -* ``%(batch_id)s`` - gets replaced by sequence number of batch +(e.g. `2020-03-28T14-45-08.237134`) -For instance:: +* ``%(batch_id)s`` - gets replaced by the batch sequence number of batch +(e.g. `2` for the second file) + +For instance, if your settings include:: FEED_STORAGE_BATCH_ITEM_COUNT=100 -Your request can be like:: +And your :command:`crawl` command line is:: scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json -The result directory tree of above can be like:: +The resulting directory tree of above can be like:: ->projectname -->dirname @@ -462,5 +465,5 @@ The result directory tree of above can be like:: --->2-filename2020-03-28T14-45-09.148903.json --->3-filename2020-03-28T14-45-10.046092.json -Where first and second files contain exactly 100 items. The last one contains +Where the first and second files contain exactly 100 items. The last one contains <= 100 items. \ No newline at end of file diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 5bc946634..4c9362f3a 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -25,6 +25,7 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values + logger = logging.getLogger(__name__) @@ -337,9 +338,9 @@ class FeedExporter: spider=spider, template_uri=slot.template_uri, )) - self.slots[idx] = None - self.slots = [slot for slot in self.slots if slot is not None] - self.slots.extend(slots) + else: + slots.append(slot) + self.slots = slots def _load_components(self, setting_prefix): conf = without_none_values(self.settings.getwithbase(setting_prefix)) From 69c005f013eb0dc000611853e66371fad17dea9d Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 14 May 2020 10:35:56 +0300 Subject: [PATCH 088/568] Documentation indent fix --- docs/topics/feed-exports.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 42c4e2267..dfeea5b7f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -444,10 +444,10 @@ based on the timestamp when the feed is being created and/or on the batch sequen Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`. * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created -(e.g. `2020-03-28T14-45-08.237134`) + (e.g. `2020-03-28T14-45-08.237134`) * ``%(batch_id)s`` - gets replaced by the batch sequence number of batch -(e.g. `2` for the second file) + (e.g. `2` for the second file) For instance, if your settings include:: From 1cdcf8b08b8f1e68c5b107b6ae39b2da1aedd245 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Fri, 15 May 2020 19:46:36 +0300 Subject: [PATCH 089/568] Minor fixes --- docs/topics/feed-exports.rst | 19 ++++++++++--------- scrapy/extensions/feedexport.py | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index dfeea5b7f..638733b6a 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -437,17 +437,18 @@ FEED_STORAGE_BATCH_ITEM_COUNT ----------------------------- Default: ``None`` -An integer number that represents the number of scraped items stored in each output -file. Whenever the number of items exceeds this setting, a new file is -created and the output is redirected to it. The name of the new file will be selected -based on the timestamp when the feed is being created and/or on the batch sequence number. -Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`. +If assigned an integer number higher than ``0``, Scrapy generates multiple output files +storing up to the specified number of items in each output file. + +When generating multiple output files, you must use at least one of the following +placeholders in :setting:`FEED_URI` to indicate how the different output file names are +generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created - (e.g. `2020-03-28T14-45-08.237134`) + (e.g. ``2020-03-28T14-45-08.237134``) * ``%(batch_id)s`` - gets replaced by the batch sequence number of batch - (e.g. `2` for the second file) + (e.g. ``2`` for the second file) For instance, if your settings include:: @@ -457,7 +458,7 @@ And your :command:`crawl` command line is:: scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json -The resulting directory tree of above can be like:: +The command line above can generate a directory tree like:: ->projectname -->dirname @@ -466,4 +467,4 @@ The resulting directory tree of above can be like:: --->3-filename2020-03-28T14-45-10.046092.json Where the first and second files contain exactly 100 items. The last one contains -<= 100 items. \ No newline at end of file +100 items or fever. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4c9362f3a..3d691c580 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -180,7 +180,7 @@ class FTPFeedStorage(BlockingFeedStorage): class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri): + def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template): self.file = file self.exporter = exporter self.storage = storage @@ -188,7 +188,7 @@ class _FeedSlot: self.batch_id = batch_id self.format = format self.store_empty = store_empty - self.template_uri = template_uri + self.uri_template = uri_template self.uri = uri # flags self.itemcount = 0 @@ -260,7 +260,7 @@ class FeedExporter: uri=uri % uri_params, feed=feed, spider=spider, - template_uri=uri, + uri_template=uri, )) def close_spider(self, spider): @@ -288,7 +288,7 @@ class FeedExporter: extra={'spider': spider})) return d - def _start_new_batch(self, batch_id, uri, feed, spider, template_uri): + def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): """ Redirect the output data stream to a new file. Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified. @@ -296,7 +296,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri + :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri """ storage = self._get_storage(uri) file = storage.open(spider) @@ -315,7 +315,7 @@ class FeedExporter: format=feed['format'], store_empty=feed['store_empty'], batch_id=batch_id, - template_uri=template_uri, + uri_template=uri_template, ) if slot.store_empty: slot.start_exporting() @@ -329,14 +329,14 @@ class FeedExporter: slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one if self.storage_batch_size and slot.itemcount == self.storage_batch_size: - uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) + uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( batch_id=slot.batch_id + 1, - uri=slot.template_uri % uri_params, - feed=self.feeds[slot.template_uri], + uri=slot.uri_template % uri_params, + feed=self.feeds[slot.uri_template], spider=spider, - template_uri=slot.template_uri, + uri_template=slot.uri_template, )) else: slots.append(slot) From 10ae1a284f759b541d086e3d1a13cda96b6e2040 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Fri, 15 May 2020 22:50:54 +0300 Subject: [PATCH 090/568] Minor fixes --- docs/topics/feed-exports.rst | 2 +- scrapy/extensions/feedexport.py | 10 +++++----- tests/test_feedexport.py | 2 +- tox.ini | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 638733b6a..6f7db20c4 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -467,4 +467,4 @@ The command line above can generate a directory tree like:: --->3-filename2020-03-28T14-45-10.046092.json Where the first and second files contain exactly 100 items. The last one contains -100 items or fever. +100 items or fewer. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3d691c580..cc26ae173 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -243,11 +243,11 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) + self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured - if not self._batch_deliveries_supported(uri): + if not self._settings_are_valid(uri): raise NotConfigured if not self._exporter_supported(feed['format']): raise NotConfigured @@ -328,7 +328,7 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one - if self.storage_batch_size and slot.itemcount == self.storage_batch_size: + if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count: uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( @@ -357,12 +357,12 @@ class FeedExporter: return True logger.error("Unknown feed format: %(format)s", {'format': format}) - def _batch_deliveries_supported(self, uri): + def _settings_are_valid(self, uri): """ If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ - if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri: + if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') return False diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d1374f291..88f9a5933 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -986,7 +986,7 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(data['csv'], b'') -class PartialDeliveriesTest(FeedExportTestBase): +class BatchDeliveriesTest(FeedExportTestBase): __test__ = True _file_mark = '_%(batch_time)s_#%(batch_id)s_' diff --git a/tox.ini b/tox.ini index 6dd944dff..7507a14a6 100644 --- a/tox.ini +++ b/tox.ini @@ -12,9 +12,9 @@ deps = -ctests/constraints.txt -rtests/requirements-py3.txt # Extras + boto3>=1.13.0 botocore>=1.3.23 Pillow>=3.4.2 - boto3>=1.13.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From a7d070f3bb350cbe1f7b580350d5f491f59d47d8 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Mon, 18 May 2020 22:25:29 +0300 Subject: [PATCH 091/568] Change log level to error --- scrapy/extensions/feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index cc26ae173..ce7fc372d 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -364,7 +364,7 @@ class FeedExporter: """ if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') + logger.error('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') return False def _storage_supported(self, uri): From 677e619d3761e6669c247786bb95822ce38c8080 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 21 May 2020 14:57:03 +0300 Subject: [PATCH 092/568] Fix too long lines --- scrapy/extensions/feedexport.py | 4 +++- tests/test_feedexport.py | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index ce7fc372d..1f745be98 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -364,7 +364,9 @@ class FeedExporter: """ if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.error('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') + logger.error( + '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified' + ) return False def _storage_supported(self, uri): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 08ee24768..fecb17e29 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1204,12 +1204,20 @@ class BatchDeliveriesTest(FeedExportTestBase): items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] formats = { - 'json': [u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), - u'[\n{"bar": "BAR1"}\n]'.encode('utf-8')], - 'xml': [u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode('latin-1'), - u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>'.encode('latin-1')], - 'csv': [u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), - u'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')], + 'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'), + '[\n{"bar": "BAR1"}\n]'.encode('utf-8')], + 'xml': [ + ( + '<?xml version="1.0" encoding="latin-1"?>\n' + '<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>' + ).encode('latin-1'), + ( + '<?xml version="1.0" encoding="latin-1"?>\n' + '<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>' + ).encode('latin-1') + ], + 'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), + 'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')], } settings = { From dd96f94e8cc1517b7021e35e46cbdc92580c6333 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Fri, 22 May 2020 23:30:33 +0300 Subject: [PATCH 093/568] Push datetime.utcnow() to its own variable --- scrapy/extensions/feedexport.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 1f745be98..45c2971a6 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -398,8 +398,9 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) - params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') - params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-') + utc_now = datetime.utcnow() + params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-') + params['batch_time'] = utc_now.isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) From 9408c77a1e16df89feeab055d3530ebad66555e1 Mon Sep 17 00:00:00 2001 From: Aditya Kumar <k.aditya00@gmail.com> Date: Sun, 31 May 2020 13:09:56 +0530 Subject: [PATCH 094/568] feat(http2): IH2EventsHandler, http2 module --- scrapy/core/http2/__init__.py | 0 scrapy/core/http2/protocol.py | 136 ++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 scrapy/core/http2/__init__.py create mode 100644 scrapy/core/http2/protocol.py diff --git a/scrapy/core/http2/__init__.py b/scrapy/core/http2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py new file mode 100644 index 000000000..4a465168f --- /dev/null +++ b/scrapy/core/http2/protocol.py @@ -0,0 +1,136 @@ +from h2.connection import H2Connection +from h2.config import H2Configuration +from h2.events import ( + ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, + StreamReset, TrailersReceived, WindowUpdated +) + +from scrapy.http import Request + +from twisted.internet.defer import maybeDeferred +from twisted.internet.protocol import Protocol + +from urllib.parse import urlparse + +from zope.interface import implementer, Interface + + +class IH2EventsHandler(Interface): + def connection_terminated(event: ConnectionTerminated): + pass + + def data_received(event: DataReceived): + pass + + def response_received(event: ResponseReceived): + pass + + def stream_ended(event: StreamEnded): + pass + + def stream_reset(event: StreamReset): + pass + + def trailers_received(event: TrailersReceived): + pass + + def window_updated(event: WindowUpdated): + pass + + +@implementer(IH2EventsHandler) +class H2ClientProtocol(Protocol): + def __init__(self): + config = H2Configuration(client_side=True) + self.conn = H2Connection(config=config) + + # List of ongoing stream id's + self.streams = [] + + def request(self, _request: Request): + url = urlparse(_request.url) + + request_headers = [ + (':method', _request.method), + (':authority', url.netloc), + (':scheme', url.scheme), + (':path', url.path), + ] + + # TODO: Check for user-agent while testing + request_headers += list(_request.headers.items()) + + # TODO: Add support for cookies here + + + + + + + def connectionMade(self): + """Called by Twisted when the connection is established. We can start + sending some data now: we should open with the connection preamble. + """ + self.conn.initiate_connection() + self.transport.write(self.conn.data_to_send()) + + def dataReceived(self, data): + events = self.conn.receive_data(data) + + self._handle_events(events) + + _data = self.conn.data_to_send() + if _data: + self.transport.write(data) + + def connectionLost(self, reason): + """Called by Twisted when the transport connection is lost. + """ + + for stream_id in self.streams: + self.conn.end_stream(stream_id) + + def _handle_events(self, events): + """Private method which acts as a bridge between the events + received from the HTTP/2 data and IH2EventsHandler + + Arguments: + events {list} -- A list of events that the remote peer + triggered by sending data + """ + for event in events: + if isinstance(event, ConnectionTerminated): + self.connection_terminated(event) + elif isinstance(event, DataReceived): + self.data_received(event) + elif isinstance(event, ResponseReceived): + self.response_received(event) + elif isinstance(event, StreamEnded): + self.stream_ended(event) + elif isinstance(event, StreamReset): + self.stream_reset(event) + elif isinstance(event, TrailersReceived): + self.trailers_received(event) + elif isinstance(event, WindowUpdated): + self.window_updated(event) + + def connection_terminated(self, event): + pass + + def data_received(self, event): + pass + + def response_received(self, event): + pass + + def stream_ended(self, event): + pass + + def stream_reset(self, event): + pass + + def trailers_received(self, event): + pass + + def window_updated(self, event): + pass From 791292334e86723a80cfd94b2877a258721f2c93 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Tue, 2 Jun 2020 09:13:31 +0530 Subject: [PATCH 095/568] chore(http2): Stream class --- scrapy/core/http2/protocol.py | 75 +++++++++++++++++++++++------------ scrapy/core/http2/stream.py | 44 ++++++++++++++++++++ 2 files changed, 94 insertions(+), 25 deletions(-) create mode 100644 scrapy/core/http2/stream.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 4a465168f..b95cb05ae 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -6,8 +6,9 @@ from h2.events import ( ) from scrapy.http import Request +from scrapy.core.http2.stream import Stream -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import Deferred from twisted.internet.protocol import Protocol from urllib.parse import urlparse @@ -40,31 +41,55 @@ class IH2EventsHandler(Interface): @implementer(IH2EventsHandler) class H2ClientProtocol(Protocol): + # TODO: Check for user-agent while testing + # TODO: Add support for cookies + # TODO: Handle priority updates + def __init__(self): config = H2Configuration(client_side=True) self.conn = H2Connection(config=config) - - # List of ongoing stream id's - self.streams = [] + + # ID of the next request stream + # Assuming each request stream creates a new response stream + # we increment by 2 for each new request stream created + self.next_stream_id = 1 + + # Streams are stored in a dictionary keyed off their stream IDs + self.streams = {} + + def _new_stream(self, headers): + """Instantiates a new Stream object + """ + stream = Stream(self.next_stream_id, headers) + + self.next_stream_id += 2 + + return stream def request(self, _request: Request): + """ + + Arguments: + _request {Request} -- [description] + """ url = urlparse(_request.url) - request_headers = [ - (':method', _request.method), - (':authority', url.netloc), - (':scheme', url.scheme), - (':path', url.path), - ] - - # TODO: Check for user-agent while testing - request_headers += list(_request.headers.items()) - - # TODO: Add support for cookies here + _request[":method"] = _request.method + # TODO: Make authority private class variable instead + # of parsing it from request url all requests to same + # host are multiplexed into one connection & a connection + # can have only 1 host at a time + _request[":authority"] = url.netloc + # TODO: Check if scheme can be 'http' for HTTP/2 ? + _request[":scheme"] = "https" + _request[":path"] = url.path + stream = self._new_stream(_request.headers) + d = stream.get_response() + return d def connectionMade(self): @@ -76,7 +101,6 @@ class H2ClientProtocol(Protocol): def dataReceived(self, data): events = self.conn.receive_data(data) - self._handle_events(events) _data = self.conn.data_to_send() @@ -85,9 +109,10 @@ class H2ClientProtocol(Protocol): def connectionLost(self, reason): """Called by Twisted when the transport connection is lost. - """ + """ - for stream_id in self.streams: + for stream_id in self.streams.keys(): + # TODO: Close each Stream instance in a clean manner self.conn.end_stream(stream_id) def _handle_events(self, events): @@ -114,23 +139,23 @@ class H2ClientProtocol(Protocol): elif isinstance(event, WindowUpdated): self.window_updated(event) - def connection_terminated(self, event): + def connection_terminated(self, event: ConnectionTerminated): pass - def data_received(self, event): + def data_received(self, event: DataReceived): pass - def response_received(self, event): + def response_received(self, event: ResponseReceived): pass - def stream_ended(self, event): + def stream_ended(self, event: StreamEnded): pass - def stream_reset(self, event): + def stream_reset(self, event: StreamReset): pass - def trailers_received(self, event): + def trailers_received(self, event: TrailersReceived): pass - def window_updated(self, event): + def window_updated(self, event: WindowUpdated): pass diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py new file mode 100644 index 000000000..558a6552c --- /dev/null +++ b/scrapy/core/http2/stream.py @@ -0,0 +1,44 @@ +from scrapy.http.headers import Headers + + +class Stream: + """Represents a single HTTP/2 Stream. + + Stream is a bidirectional flow of bytes within an established connection, + which may carry one or more messages. Handles the tranfer of HTTP Headers + and Data frames. + """ + + def __init__(self, stream_id, headers): + """ + Arguments: + stream_id {int} -- For one HTTP/2 connection each stream is + uniquely identified by a single integer + headers {Headers} -- HTTP request headers + """ + + # Headers received after sending the request + self.response_headers = Headers({}) + + # Headers which are send with the request + # These cannot be modified any furthur + self._request_headers = headers + + # TODO: Add canceller for the Deferred below + self._deferred_response = Deferred() + + def get_response(self): + """Simply return a Deferred which fires when response + from the asynchronous request is available + + Returns: + Deferred -- Calls the callback when the response is + avaialble + """ + return self._deferred_response + + def receive_data(self, data): + pass + + def receive_headers(self, headers): + pass From 0cabf406d5c50060ac2da4e5c76d704606bee990 Mon Sep 17 00:00:00 2001 From: Matthias Meschede <MMesch@users.noreply.github.com> Date: Mon, 1 Jun 2020 17:41:52 +0200 Subject: [PATCH 096/568] set write permission to startproject folder --- scrapy/commands/startproject.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index ebe3a9c2c..cfaa25154 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -4,6 +4,7 @@ import string from importlib import import_module from os.path import join, exists, abspath from shutil import ignore_patterns, move, copy2, copystat +import stat import scrapy from scrapy.commands import ScrapyCommand @@ -79,6 +80,28 @@ class Command(ScrapyCommand): copy2(srcname, dstname) copystat(src, dst) + def _set_rw_permissions(self, path): + """ + Sets permissions of a directory tree to +rw and +rwx for folders. + This is necessary if the start template files come without write + permissions. + """ + mode_rw = (stat.S_IRUSR + | stat.S_IWUSR + | stat.S_IRGRP + | stat.S_IROTH) + + mode_x = (stat.S_IXUSR + | stat.S_IXGRP + | stat.S_IXOTH) + + os.chmod(path, mode_rw | mode_x) + for root, dirs, files in os.walk(path): + for dir in dirs: + os.chmod(join(root, dir), mode_rw | mode_x) + for file in files: + os.chmod(join(root, file), mode_rw) + def run(self, args, opts): if len(args) not in (1, 2): raise UsageError() @@ -99,6 +122,9 @@ class Command(ScrapyCommand): return self._copytree(self.templates_dir, abspath(project_dir)) + + self._set_rw_permissions(abspath(project_dir)) + move(join(project_dir, 'module'), join(project_dir, project_name)) for paths in TEMPLATES_TO_RENDER: path = join(*paths) From 2df3b54c7d2666a95793ea6788e88d1ae945e0f5 Mon Sep 17 00:00:00 2001 From: Matthias Meschede <MMesch@users.noreply.github.com> Date: Fri, 5 Jun 2020 09:29:05 +0200 Subject: [PATCH 097/568] refactor --- scrapy/commands/startproject.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index cfaa25154..3d1f5914f 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -79,6 +79,7 @@ class Command(ScrapyCommand): else: copy2(srcname, dstname) copystat(src, dst) + self._set_rw_permissions(dst) def _set_rw_permissions(self, path): """ @@ -123,8 +124,6 @@ class Command(ScrapyCommand): self._copytree(self.templates_dir, abspath(project_dir)) - self._set_rw_permissions(abspath(project_dir)) - move(join(project_dir, 'module'), join(project_dir, project_name)) for paths in TEMPLATES_TO_RENDER: path = join(*paths) From 9ff9caecadf8215ce75b0ad4231b8289bca168fe Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Sun, 7 Jun 2020 14:04:53 +0530 Subject: [PATCH 098/568] feat(http2): support for GET requests --- scrapy/core/http2/protocol.py | 126 ++++++++++++++++++++++------------ scrapy/core/http2/stream.py | 71 +++++++++++++++---- 2 files changed, 139 insertions(+), 58 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index b95cb05ae..847e74f97 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,20 +1,19 @@ -from h2.connection import H2Connection +import logging + from h2.config import H2Configuration +from h2.connection import H2Connection from h2.events import ( - ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, - StreamReset, TrailersReceived, WindowUpdated + ConnectionTerminated, DataReceived, ResponseReceived, RemoteSettingsChanged, + StreamEnded, StreamReset, TrailersReceived, WindowUpdated ) - -from scrapy.http import Request -from scrapy.core.http2.stream import Stream - -from twisted.internet.defer import Deferred -from twisted.internet.protocol import Protocol - -from urllib.parse import urlparse - +from twisted.internet.protocol import connectionDone, Protocol from zope.interface import implementer, Interface +from scrapy.core.http2.stream import Stream +from scrapy.http import Request + +LOGGER = logging.getLogger(__name__) + class IH2EventsHandler(Interface): def connection_terminated(event: ConnectionTerminated): @@ -26,6 +25,9 @@ class IH2EventsHandler(Interface): def response_received(event: ResponseReceived): pass + def remote_settings_changed(event: RemoteSettingsChanged): + pass + def stream_ended(event: StreamEnded): pass @@ -46,7 +48,7 @@ class H2ClientProtocol(Protocol): # TODO: Handle priority updates def __init__(self): - config = H2Configuration(client_side=True) + config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) # ID of the next request stream @@ -57,60 +59,69 @@ class H2ClientProtocol(Protocol): # Streams are stored in a dictionary keyed off their stream IDs self.streams = {} - def _new_stream(self, headers): + # Boolean to keep track the connection is made + # If requests are received before connection is made + # we keep all requests in a pool and send them as the connection + # is made + self.is_connection_made = False + self._pending_request_stream_pool = [] + + def _new_stream(self, request: Request): """Instantiates a new Stream object """ - stream = Stream(self.next_stream_id, headers) - + stream = Stream(self.next_stream_id, request, self) self.next_stream_id += 2 + self.streams[stream.stream_id] = stream return stream + def _write_to_transport(self): + """ Write data to the underlying transport connection + from the HTTP2 connection instance if any + """ + data = self.conn.data_to_send() + if data: + self.transport.write(data) + def request(self, _request: Request): - """ - - Arguments: - _request {Request} -- [description] - """ - url = urlparse(_request.url) - - _request[":method"] = _request.method - - # TODO: Make authority private class variable instead - # of parsing it from request url all requests to same - # host are multiplexed into one connection & a connection - # can have only 1 host at a time - _request[":authority"] = url.netloc - - # TODO: Check if scheme can be 'http' for HTTP/2 ? - _request[":scheme"] = "https" - _request[":path"] = url.path - - stream = self._new_stream(_request.headers) + stream = self._new_stream(_request) d = stream.get_response() - return d + # If connection is not yet established then add the + # stream to pool or initiate request + if self.is_connection_made: + stream.initiate_request() + else: + self._pending_request_stream_pool.append(stream) + return d def connectionMade(self): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ + LOGGER.info("Connection made to {}".format(self.transport)) self.conn.initiate_connection() - self.transport.write(self.conn.data_to_send()) + self._write_to_transport() + + self.is_connection_made = True + + # Initiate all pending requests + for stream in self._pending_request_stream_pool: + assert isinstance(stream, Stream) + stream.initiate_request() + + self._pending_request_stream_pool.clear() def dataReceived(self, data): events = self.conn.receive_data(data) self._handle_events(events) + self._write_to_transport() - _data = self.conn.data_to_send() - if _data: - self.transport.write(data) + def connectionLost(self, reason=connectionDone): - def connectionLost(self, reason): """Called by Twisted when the transport connection is lost. """ - for stream_id in self.streams.keys(): # TODO: Close each Stream instance in a clean manner self.conn.end_stream(stream_id) @@ -138,18 +149,43 @@ class H2ClientProtocol(Protocol): self.trailers_received(event) elif isinstance(event, WindowUpdated): self.window_updated(event) + elif isinstance(event, RemoteSettingsChanged): + self.remote_settings_changed(event) + + def send_headers(self, stream_id, headers): + """ Send the headers for a given stream to the resource + Initiates a new connection hence. + + Arguments: + stream_id {int} -- Valid stream id + headers {List[Tuple[str, str]]} -- Headers of the request + """ + if stream_id in self.streams: + self.conn.send_headers(stream_id, headers, end_stream=True) + self._write_to_transport() + else: + pass def connection_terminated(self, event: ConnectionTerminated): pass def data_received(self, event: DataReceived): - pass + stream_id = event.stream_id + # TODO: Stream do not exist in self.streams dict + self.streams[stream_id].receive_data(event.data) def response_received(self, event: ResponseReceived): + stream_id = event.stream_id + # TODO: Stream do not exist in self.streams dict + self.streams[stream_id].receive_headers(event.headers) + + def remote_settings_changed(self, event: RemoteSettingsChanged): pass def stream_ended(self, event: StreamEnded): - pass + stream_id = event.stream_id + # TODO: Stream do not exist in self.streams dict + self.streams[stream_id].end_stream() def stream_reset(self, event: StreamReset): pass diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 558a6552c..d2a9f02fa 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,3 +1,8 @@ +from urllib.parse import urlparse + +from twisted.internet.defer import Deferred + +from scrapy.http import Request, Response from scrapy.http.headers import Headers @@ -5,24 +10,30 @@ class Stream: """Represents a single HTTP/2 Stream. Stream is a bidirectional flow of bytes within an established connection, - which may carry one or more messages. Handles the tranfer of HTTP Headers + which may carry one or more messages. Handles the transfer of HTTP Headers and Data frames. + + Role of this class is to + 1. Combine all the data frames """ - def __init__(self, stream_id, headers): + def __init__(self, stream_id: int, request: Request, connection): """ Arguments: stream_id {int} -- For one HTTP/2 connection each stream is uniquely identified by a single integer - headers {Headers} -- HTTP request headers + request {Request} -- HTTP request + connection {H2ClientProtocol} -- HTTP/2 connection this stream belongs to """ - # Headers received after sending the request - self.response_headers = Headers({}) + self.stream_id = stream_id + self._request = request + self._conn = connection - # Headers which are send with the request - # These cannot be modified any furthur - self._request_headers = headers + self._response_data = b"" + + # Headers received after sending the request + self._response_headers = Headers({}) # TODO: Add canceller for the Deferred below self._deferred_response = Deferred() @@ -32,13 +43,47 @@ class Stream: from the asynchronous request is available Returns: - Deferred -- Calls the callback when the response is - avaialble + Deferred -- Calls the callback passing the response """ return self._deferred_response - def receive_data(self, data): - pass + def initiate_request(self): + http2_request_headers = [] + for name, value in self._request.headers.items(): + http2_request_headers.append((name, value)) + + url = urlparse(self._request.url) + http2_request_headers += [ + (":method", self._request.method), + (":authority", url.netloc), + + # TODO: Check if scheme can be "http" for HTTP/2 ? + (":scheme", "https"), + (":path", url.path) + ] + + self._conn.send_headers(self.stream_id, http2_request_headers) + + def receive_data(self, data: bytes): + self._response_data += data def receive_headers(self, headers): - pass + for name, value in headers: + self._response_headers[name] = value + + def end_stream(self): + """Stream is ended by the resource hence no further + data or headers should be expected on this stream. + + We will call the response deferred callback passing + the response object + """ + # TODO: Set flags, certificate, ip_address + response = Response( + url=self._request.url, + status=self._response_headers[":status"], + headers=self._response_headers, + body=self._response_data, + request=self._request + ) + self._deferred_response.callback(response) From 76a2cbf0ff7a060833812704a0416ba617ddc8b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 9 Jun 2020 21:30:19 +0200 Subject: [PATCH 099/568] Apply minor style changes --- scrapy/commands/startproject.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 3d1f5914f..852281959 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,10 +1,10 @@ import re import os +import stat import string from importlib import import_module from os.path import join, exists, abspath from shutil import ignore_patterns, move, copy2, copystat -import stat import scrapy from scrapy.commands import ScrapyCommand @@ -123,7 +123,6 @@ class Command(ScrapyCommand): return self._copytree(self.templates_dir, abspath(project_dir)) - move(join(project_dir, 'module'), join(project_dir, project_name)) for paths in TEMPLATES_TO_RENDER: path = join(*paths) From 8b549392f924ddad9536e55c6120638daf688dfd Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 11 Jun 2020 09:53:59 -0300 Subject: [PATCH 100/568] Bump minimum Python version to 3.5.2 (#4615) --- .travis.yml | 6 ++---- README.rst | 2 +- docs/faq.rst | 2 +- docs/intro/install.rst | 2 +- scrapy/__init__.py | 4 ++-- setup.py | 2 +- 6 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index d6ec88e06..e44f85237 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,11 +18,9 @@ matrix: - env: TOXENV=pypy3 - env: TOXENV=pinned - python: 3.5.1 - dist: trusty + python: 3.5.2 - env: TOXENV=asyncio - python: 3.5.1 # We use additional code to support 3.5.3 and earlier - dist: trusty + python: 3.5.2 # We use additional code to support 3.5.3 and earlier - env: TOXENV=py python: 3.5 - env: TOXENV=asyncio diff --git a/README.rst b/README.rst index fd84e127e..0e3939e9b 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ including a list of features. Requirements ============ -* Python 3.5.1+ +* Python 3.5.2+ * Works on Linux, Windows, macOS, BSD Install diff --git a/docs/faq.rst b/docs/faq.rst index c06cb945b..9cdb7d09d 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars What Python versions does Scrapy support? ----------------------------------------- -Scrapy is supported under Python 3.5.1+ +Scrapy is supported under Python 3.5.2+ under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). Python 3 support was added in Scrapy 1.1. PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 4af80d801..fb64d443c 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,7 +7,7 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 3.5.1 or above under CPython (default Python +Scrapy runs on Python 3.5.2 or above under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). If you're using `Anaconda`_ or `Miniconda`_, you can install the package from diff --git a/scrapy/__init__.py b/scrapy/__init__.py index e791deaa6..f0259a9b7 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version -if sys.version_info < (3, 5): - print("Scrapy %s requires Python 3.5" % __version__) +if sys.version_info < (3, 5, 2): + print("Scrapy %s requires Python 3.5.2" % __version__) sys.exit(1) diff --git a/setup.py b/setup.py index 1b3c6771a..71dc3232d 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=3.5', + python_requires='>=3.5.2', install_requires=[ 'Twisted>=17.9.0', 'cryptography>=2.0', From d09ccf8d3b2932d9393e6ce4a20c46befdd43acf Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Sat, 13 Jun 2020 20:40:01 +0530 Subject: [PATCH 101/568] feat(http2): support for POST requests BREAKING CHANGES - Request is sent successfully with its Response received as well. However, the StreamEnded event is not received which do not fires the response deferred --- scrapy/core/http2/protocol.py | 108 +++++++++++++++++----------------- scrapy/core/http2/stream.py | 79 +++++++++++++++++++++++-- 2 files changed, 127 insertions(+), 60 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 847e74f97..6d926b100 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,4 +1,5 @@ import logging +from typing import Dict, List from h2.config import H2Configuration from h2.connection import H2Connection @@ -7,45 +8,19 @@ from h2.events import ( StreamEnded, StreamReset, TrailersReceived, WindowUpdated ) from twisted.internet.protocol import connectionDone, Protocol -from zope.interface import implementer, Interface from scrapy.core.http2.stream import Stream from scrapy.http import Request LOGGER = logging.getLogger(__name__) +LOGGER.debug = print -class IH2EventsHandler(Interface): - def connection_terminated(event: ConnectionTerminated): - pass - - def data_received(event: DataReceived): - pass - - def response_received(event: ResponseReceived): - pass - - def remote_settings_changed(event: RemoteSettingsChanged): - pass - - def stream_ended(event: StreamEnded): - pass - - def stream_reset(event: StreamReset): - pass - - def trailers_received(event: TrailersReceived): - pass - - def window_updated(event: WindowUpdated): - pass - - -@implementer(IH2EventsHandler) class H2ClientProtocol(Protocol): - # TODO: Check for user-agent while testing - # TODO: Add support for cookies - # TODO: Handle priority updates + # TODO: + # 1. Check for user-agent while testing + # 2. Add support for cookies + # 3. Handle priority updates def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') @@ -57,14 +32,14 @@ class H2ClientProtocol(Protocol): self.next_stream_id = 1 # Streams are stored in a dictionary keyed off their stream IDs - self.streams = {} + self.streams: Dict[int, Stream] = {} # Boolean to keep track the connection is made # If requests are received before connection is made # we keep all requests in a pool and send them as the connection # is made self.is_connection_made = False - self._pending_request_stream_pool = [] + self._pending_request_stream_pool: List[Stream] = [] def _new_stream(self, request: Request): """Instantiates a new Stream object @@ -100,29 +75,27 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ - LOGGER.info("Connection made to {}".format(self.transport)) + LOGGER.debug("Connection made to {}".format(self.transport)) self.conn.initiate_connection() self._write_to_transport() self.is_connection_made = True - # Initiate all pending requests - for stream in self._pending_request_stream_pool: - assert isinstance(stream, Stream) - stream.initiate_request() - - self._pending_request_stream_pool.clear() - def dataReceived(self, data): events = self.conn.receive_data(data) self._handle_events(events) self._write_to_transport() def connectionLost(self, reason=connectionDone): - """Called by Twisted when the transport connection is lost. """ - for stream_id in self.streams.keys(): + LOGGER.debug(f"connectionLost {reason}") + stream_ids = list(self.streams.keys()) + + for stream in self._pending_request_stream_pool: + stream_ids.remove(stream.stream_id) + + for stream_id in stream_ids: # TODO: Close each Stream instance in a clean manner self.conn.end_stream(stream_id) @@ -135,6 +108,7 @@ class H2ClientProtocol(Protocol): triggered by sending data """ for event in events: + LOGGER.debug(event) if isinstance(event, ConnectionTerminated): self.connection_terminated(event) elif isinstance(event, DataReceived): @@ -153,38 +127,62 @@ class H2ClientProtocol(Protocol): self.remote_settings_changed(event) def send_headers(self, stream_id, headers): - """ Send the headers for a given stream to the resource + """Send the headers for a given stream to the resource Initiates a new connection hence. + This function is wrapper for :func:`~h2.connection.H2Connection.send_headers` Arguments: stream_id {int} -- Valid stream id headers {List[Tuple[str, str]]} -- Headers of the request """ - if stream_id in self.streams: - self.conn.send_headers(stream_id, headers, end_stream=True) - self._write_to_transport() - else: - pass + LOGGER.debug(f'Send Headers: stream_id={stream_id} headers={headers}') + self.conn.send_headers(stream_id, headers, end_stream=False) + def send_data(self, stream_id, data): + """Send the data for a given stream to the resource. + Requires request headers to be sent at least once before this + function is called. + This function is wrapper for :func:`~h2.connection.H2Connection.send_data` + + Arguments: + stream_id {int} -- Valid stream id + data {bytes} -- The data to send on the stream. + """ + LOGGER.debug(f"Send Data: stream_id={stream_id} data={data}") + self.conn.send_data(stream_id, data, end_stream=False) + + def end_stream(self, stream_id): + """End the given stream. + This function is wrapper for :func:`~h2.connection.H2Connection.end_stream` + + Arguments: + stream_id {int} - Valid stream id + """ + LOGGER.debug(f"End Stream: stream_id={stream_id}") + self.conn.end_stream(stream_id) + + # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated): pass def data_received(self, event: DataReceived): stream_id = event.stream_id - # TODO: Stream do not exist in self.streams dict self.streams[stream_id].receive_data(event.data) def response_received(self, event: ResponseReceived): stream_id = event.stream_id - # TODO: Stream do not exist in self.streams dict self.streams[stream_id].receive_headers(event.headers) def remote_settings_changed(self, event: RemoteSettingsChanged): - pass + # TODO: handle MAX_CONCURRENT_STREAMS + # Initiate all pending requests + for stream in self._pending_request_stream_pool: + stream.initiate_request() + + self._pending_request_stream_pool.clear() def stream_ended(self, event: StreamEnded): stream_id = event.stream_id - # TODO: Stream do not exist in self.streams dict self.streams[stream_id].end_stream() def stream_reset(self, event: StreamReset): @@ -194,4 +192,6 @@ class H2ClientProtocol(Protocol): pass def window_updated(self, event: WindowUpdated): - pass + stream_id = event.stream_id + if stream_id != 0: + self.streams[stream_id].window_updated() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index d2a9f02fa..ab5a0fc88 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -23,13 +23,23 @@ class Stream: stream_id {int} -- For one HTTP/2 connection each stream is uniquely identified by a single integer request {Request} -- HTTP request - connection {H2ClientProtocol} -- HTTP/2 connection this stream belongs to + connection {H2Connection} -- HTTP/2 connection this stream belongs to. """ - self.stream_id = stream_id self._request = request - self._conn = connection + self._client_protocol = connection + self._request_body = self._request.body + self.content_length = 0 if self._request_body is None else len(self._request_body) + + # Each time we send a data frame, we will decrease value by the amount send. + self.remaining_content_length = self.content_length + + # Flag to keep track whether we have ended this stream + self.stream_ended = True + + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. self._response_data = b"" # Headers received after sending the request @@ -59,10 +69,67 @@ class Stream: # TODO: Check if scheme can be "http" for HTTP/2 ? (":scheme", "https"), - (":path", url.path) + (":path", url.path), + # ("Content-Length", str(self.content_length)) + + # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers + # are sent for request having body ] - self._conn.send_headers(self.stream_id, http2_request_headers) + self._client_protocol.send_headers(self.stream_id, http2_request_headers) + self.send_data() + + def send_data(self): + """Called immediately after the headers are sent. Here we send all the + data as part of the request. + + If the content length is 0 initially then we end the stream immediately and + wait for response data. + """ + + # TODO: + # 1. Add test for sending very large data + # 2. Add test for small data + # 3. Both (1) and (2) should be tested for + # 3.1 Large number of request + # 3.2 Small number of requests + + # Firstly, check what the flow control window is for current stream. + window_size = self._client_protocol.conn.local_flow_control_window(stream_id=self.stream_id) + + # Next, check what the maximum frame size is. + max_frame_size = self._client_protocol.conn.max_outbound_frame_size + + # We will send no more than the window size or the remaining file size + # of data in this call, whichever is smaller. + bytes_to_send = min(window_size, self.remaining_content_length) + + # We now need to send a number of data frames. + while bytes_to_send > 0: + chunk_size = min(bytes_to_send, max_frame_size) + + data_chunk_start = self.content_length - self.remaining_content_length + data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size] + + self._client_protocol.send_data(self.stream_id, data_chunk, end_stream=False) + + bytes_to_send = max(0, bytes_to_send - chunk_size) + self.remaining_content_length = max(0, self.remaining_content_length - chunk_size) + + # End the stream if no more data has to be send + if self.remaining_content_length == 0: + self._client_protocol.end_stream(self.stream_id) + else: + # TODO: Continue from here :) + pass + + def window_updated(self): + """Flow control window size was changed. + Send data that earlier could not be sent as we were + blocked behind the flow control. + """ + if self.remaining_content_length > 0 and not self.stream_ended: + self.send_data() def receive_data(self, data: bytes): self._response_data += data @@ -72,7 +139,7 @@ class Stream: self._response_headers[name] = value def end_stream(self): - """Stream is ended by the resource hence no further + """Stream is ended by the server hence no further data or headers should be expected on this stream. We will call the response deferred callback passing From d06bb12e351f61ee20c28626b003f4b59fd689f7 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Sat, 13 Jun 2020 22:29:16 +0530 Subject: [PATCH 102/568] refactor: move H2Connection instance to stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove all wrapper funtions made such that stream can send header/data to H2Connection as they were not necessary BREAKING CHANGES Looks like, for small set of response data the StreamEnded event is emitted and everything works well -- tested for both GET & POST request. Maybe some issue with window size and/or flow control as when the response data needs to be broken into separate chunks -- not all chunks are received everytime which leads to indefinite waiting for next data chunk and the connection is lost due to timeout. 😥 Working on setting up testing environment now. After testing is setup I'll debug the above bug furthur. --- scrapy/core/http2/protocol.py | 74 ++++++++--------------------------- scrapy/core/http2/stream.py | 16 ++++---- 2 files changed, 24 insertions(+), 66 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 6d926b100..4036dfb3e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -4,8 +4,7 @@ from typing import Dict, List from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - ConnectionTerminated, DataReceived, ResponseReceived, RemoteSettingsChanged, - StreamEnded, StreamReset, TrailersReceived, WindowUpdated + ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, StreamReset, WindowUpdated ) from twisted.internet.protocol import connectionDone, Protocol @@ -13,7 +12,6 @@ from scrapy.core.http2.stream import Stream from scrapy.http import Request LOGGER = logging.getLogger(__name__) -LOGGER.debug = print class H2ClientProtocol(Protocol): @@ -44,19 +42,27 @@ class H2ClientProtocol(Protocol): def _new_stream(self, request: Request): """Instantiates a new Stream object """ - stream = Stream(self.next_stream_id, request, self) + stream = Stream(self.next_stream_id, request, self.conn) self.next_stream_id += 2 self.streams[stream.stream_id] = stream return stream + def _send_pending_requests(self): + # TODO: handle MAX_CONCURRENT_STREAMS + # Initiate all pending requests + for stream in self._pending_request_stream_pool: + stream.initiate_request() + self._write_to_transport() + + self._pending_request_stream_pool.clear() + def _write_to_transport(self): """ Write data to the underlying transport connection from the HTTP2 connection instance if any """ data = self.conn.data_to_send() - if data: - self.transport.write(data) + self.transport.write(data) def request(self, _request: Request): stream = self._new_stream(_request) @@ -75,10 +81,11 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ - LOGGER.debug("Connection made to {}".format(self.transport)) self.conn.initiate_connection() self._write_to_transport() + self._send_pending_requests() + self.is_connection_made = True def dataReceived(self, data): @@ -89,7 +96,6 @@ class H2ClientProtocol(Protocol): def connectionLost(self, reason=connectionDone): """Called by Twisted when the transport connection is lost. """ - LOGGER.debug(f"connectionLost {reason}") stream_ids = list(self.streams.keys()) for stream in self._pending_request_stream_pool: @@ -119,47 +125,10 @@ class H2ClientProtocol(Protocol): self.stream_ended(event) elif isinstance(event, StreamReset): self.stream_reset(event) - elif isinstance(event, TrailersReceived): - self.trailers_received(event) elif isinstance(event, WindowUpdated): self.window_updated(event) - elif isinstance(event, RemoteSettingsChanged): - self.remote_settings_changed(event) - - def send_headers(self, stream_id, headers): - """Send the headers for a given stream to the resource - Initiates a new connection hence. - This function is wrapper for :func:`~h2.connection.H2Connection.send_headers` - - Arguments: - stream_id {int} -- Valid stream id - headers {List[Tuple[str, str]]} -- Headers of the request - """ - LOGGER.debug(f'Send Headers: stream_id={stream_id} headers={headers}') - self.conn.send_headers(stream_id, headers, end_stream=False) - - def send_data(self, stream_id, data): - """Send the data for a given stream to the resource. - Requires request headers to be sent at least once before this - function is called. - This function is wrapper for :func:`~h2.connection.H2Connection.send_data` - - Arguments: - stream_id {int} -- Valid stream id - data {bytes} -- The data to send on the stream. - """ - LOGGER.debug(f"Send Data: stream_id={stream_id} data={data}") - self.conn.send_data(stream_id, data, end_stream=False) - - def end_stream(self, stream_id): - """End the given stream. - This function is wrapper for :func:`~h2.connection.H2Connection.end_stream` - - Arguments: - stream_id {int} - Valid stream id - """ - LOGGER.debug(f"End Stream: stream_id={stream_id}") - self.conn.end_stream(stream_id) + else: + LOGGER.info("Received unhandled event {}".format(event)) # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated): @@ -173,14 +142,6 @@ class H2ClientProtocol(Protocol): stream_id = event.stream_id self.streams[stream_id].receive_headers(event.headers) - def remote_settings_changed(self, event: RemoteSettingsChanged): - # TODO: handle MAX_CONCURRENT_STREAMS - # Initiate all pending requests - for stream in self._pending_request_stream_pool: - stream.initiate_request() - - self._pending_request_stream_pool.clear() - def stream_ended(self, event: StreamEnded): stream_id = event.stream_id self.streams[stream_id].end_stream() @@ -188,9 +149,6 @@ class H2ClientProtocol(Protocol): def stream_reset(self, event: StreamReset): pass - def trailers_received(self, event: TrailersReceived): - pass - def window_updated(self, event: WindowUpdated): stream_id = event.stream_id if stream_id != 0: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index ab5a0fc88..b37755042 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,5 +1,6 @@ from urllib.parse import urlparse +from h2.connection import H2Connection from twisted.internet.defer import Deferred from scrapy.http import Request, Response @@ -17,7 +18,7 @@ class Stream: 1. Combine all the data frames """ - def __init__(self, stream_id: int, request: Request, connection): + def __init__(self, stream_id: int, request: Request, connection: H2Connection): """ Arguments: stream_id {int} -- For one HTTP/2 connection each stream is @@ -27,7 +28,7 @@ class Stream: """ self.stream_id = stream_id self._request = request - self._client_protocol = connection + self._conn = connection self._request_body = self._request.body self.content_length = 0 if self._request_body is None else len(self._request_body) @@ -70,13 +71,12 @@ class Stream: # TODO: Check if scheme can be "http" for HTTP/2 ? (":scheme", "https"), (":path", url.path), - # ("Content-Length", str(self.content_length)) # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers # are sent for request having body ] - self._client_protocol.send_headers(self.stream_id, http2_request_headers) + self._conn.send_headers(self.stream_id, http2_request_headers, end_stream=False) self.send_data() def send_data(self): @@ -95,10 +95,10 @@ class Stream: # 3.2 Small number of requests # Firstly, check what the flow control window is for current stream. - window_size = self._client_protocol.conn.local_flow_control_window(stream_id=self.stream_id) + window_size = self._conn.local_flow_control_window(stream_id=self.stream_id) # Next, check what the maximum frame size is. - max_frame_size = self._client_protocol.conn.max_outbound_frame_size + max_frame_size = self._conn.max_outbound_frame_size # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. @@ -111,14 +111,14 @@ class Stream: data_chunk_start = self.content_length - self.remaining_content_length data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size] - self._client_protocol.send_data(self.stream_id, data_chunk, end_stream=False) + self._conn.send_data(self.stream_id, data_chunk, end_stream=False) bytes_to_send = max(0, bytes_to_send - chunk_size) self.remaining_content_length = max(0, self.remaining_content_length - chunk_size) # End the stream if no more data has to be send if self.remaining_content_length == 0: - self._client_protocol.end_stream(self.stream_id) + self._conn.end_stream(self.stream_id) else: # TODO: Continue from here :) pass From ec98dabfab60283303a9208ccd8177d9f995ba72 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Sun, 14 Jun 2020 06:45:27 -0300 Subject: [PATCH 103/568] Support for dataclass and attrs items (#3881) --- docs/conf.py | 1 + docs/faq.rst | 8 +- docs/topics/architecture.rst | 2 +- docs/topics/coroutines.rst | 13 +- docs/topics/exporters.rst | 34 ++- docs/topics/feed-exports.rst | 4 +- docs/topics/item-pipeline.rst | 54 ++-- docs/topics/items.rst | 241 ++++++++++++++---- docs/topics/leaks.rst | 15 +- docs/topics/loaders.rst | 53 ++-- docs/topics/media-pipeline.rst | 33 ++- docs/topics/settings.rst | 4 +- docs/topics/signals.rst | 10 +- docs/topics/spider-middleware.rst | 13 +- docs/topics/spiders.rst | 18 +- scrapy/commands/parse.py | 6 +- scrapy/contracts/default.py | 28 +- scrapy/core/scraper.py | 27 +- scrapy/exporters.py | 38 +-- scrapy/item.py | 2 +- scrapy/loader/__init__.py | 18 +- scrapy/pipelines/files.py | 23 +- scrapy/pipelines/images.py | 19 +- scrapy/shell.py | 11 +- scrapy/spiders/feed.py | 2 +- .../project/module/middlewares.py.tmpl | 8 +- .../project/module/pipelines.py.tmpl | 4 + scrapy/utils/serialize.py | 6 +- setup.py | 1 + tests/requirements-py3.txt | 2 + tests/test_engine.py | 40 ++- tests/test_loader.py | 47 +++- tests/test_pipeline_files.py | 144 ++++++++--- tests/test_pipeline_images.py | 129 +++++++--- tests/test_utils_serialize.py | 43 +++- tox.ini | 3 +- 36 files changed, 753 insertions(+), 351 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 29b2fc406..86734fae7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -281,6 +281,7 @@ coverage_ignore_pyobjects = [ # ------------------------------------- intersphinx_mapping = { + 'attrs': ('https://www.attrs.org/en/stable/', None), 'coverage': ('https://coverage.readthedocs.io/en/stable', None), 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), 'pytest': ('https://docs.pytest.org/en/latest', None), diff --git a/docs/faq.rst b/docs/faq.rst index 9cdb7d09d..d5ea3cb87 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -342,15 +342,15 @@ method for this purpose. For example:: from copy import deepcopy - from scrapy.item import Item - + from itemadapter import is_item, ItemAdapter class MultiplyItemsMiddleware: def process_spider_output(self, response, result, spider): for item in result: - if isinstance(item, (Item, dict)): - for _ in range(item['multiply_by']): + if is_item(item): + adapter = ItemAdapter(item) + for _ in range(adapter['multiply_by']): yield deepcopy(item) Does Scrapy support IPv6 addresses? diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index ae25dfa2f..074c59241 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -104,7 +104,7 @@ Spiders ------- Spiders are custom classes written by Scrapy users to parse responses and -extract items (aka scraped items) from them or additional requests to +extract :ref:`items <topics-items>` from them or additional requests to follow. For more information see :ref:`topics-spiders`. .. _component-pipelines: diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 7a9ecd4d5..a0952d323 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -53,21 +53,28 @@ There are several use cases for coroutines in Scrapy. Code that would return Deferreds when written for previous Scrapy versions, such as downloader middlewares and signal handlers, can be rewritten to be shorter and cleaner:: + from itemadapter import ItemAdapter + class DbPipeline: def _update_item(self, data, item): - item['field'] = data + adapter = ItemAdapter(item) + adapter['field'] = data return item def process_item(self, item, spider): - dfd = db.get_some_data(item['id']) + adapter = ItemAdapter(item) + dfd = db.get_some_data(adapter['id']) dfd.addCallback(self._update_item, item) return dfd becomes:: + from itemadapter import ItemAdapter + class DbPipeline: async def process_item(self, item, spider): - item['field'] = await db.get_some_data(item['id']) + adapter = ItemAdapter(item) + adapter['field'] = await db.get_some_data(adapter['id']) return item Coroutines may be used to call asynchronous code. This includes other diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 7daf25ab3..e5c99e5b1 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -40,6 +40,7 @@ Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple Item Exporters to group scraped items to different files according to the value of one of their fields:: + from itemadapter import ItemAdapter from scrapy.exporters import XmlItemExporter class PerYearXmlExportPipeline: @@ -53,7 +54,8 @@ value of one of their fields:: exporter.finish_exporting() def _exporter_for_item(self, item): - year = item['year'] + adapter = ItemAdapter(item) + year = adapter['year'] if year not in self.year_to_exporter: f = open('{}.xml'.format(year), 'wb') exporter = XmlItemExporter(f) @@ -167,9 +169,10 @@ BaseItemExporter value unchanged except for ``unicode`` values which are encoded to ``str`` using the encoding declared in the :attr:`encoding` attribute. - :param field: the field being serialized. If a raw dict is being - exported (not :class:`~.Item`) *field* value is an empty dict. - :type field: :class:`~scrapy.item.Field` object or an empty dict + :param field: the field being serialized. If the source :ref:`item object + <item-types>` does not define field metadata, *field* is an empty + :class:`dict`. + :type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance :param name: the name of the field being serialized :type name: str @@ -192,14 +195,17 @@ BaseItemExporter .. attribute:: fields_to_export - A list with the name of the fields that will be exported, or None if you - want to export all fields. Defaults to None. + A list with the name of the fields that will be exported, or ``None`` if + you want to export all fields. Defaults to ``None``. Some exporters (like :class:`CsvItemExporter`) respect the order of the fields defined in this attribute. - Some exporters may require fields_to_export list in order to export the - data properly when spiders return dicts (not :class:`~Item` instances). + When using :ref:`item objects <item-types>` that do not expose all their + possible fields, exporters that do not support exporting a different + subset of fields per item will only export the fields found in the first + item exported. Use ``fields_to_export`` to define all the fields to be + exported. .. attribute:: export_empty_fields @@ -238,7 +244,7 @@ XmlItemExporter .. class:: XmlItemExporter(file, item_element='item', root_element='items', **kwargs) - Exports Items in XML format to the specified file object. + Exports items in XML format to the specified file object. :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) @@ -292,7 +298,7 @@ CsvItemExporter .. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs) - Exports Items in CSV format to the given file-like object. If the + Exports items in CSV format to the given file-like object. If the :attr:`fields_to_export` attribute is set, it will be used to define the CSV columns and their order. The :attr:`export_empty_fields` attribute has no effect on this exporter. @@ -325,7 +331,7 @@ PickleItemExporter .. class:: PickleItemExporter(file, protocol=0, **kwargs) - Exports Items in pickle format to the given file-like object. + Exports items in pickle format to the given file-like object. :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) @@ -345,7 +351,7 @@ PprintItemExporter .. class:: PprintItemExporter(file, **kwargs) - Exports Items in pretty print format to the specified file object. + Exports items in pretty print format to the specified file object. :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) @@ -365,7 +371,7 @@ JsonItemExporter .. class:: JsonItemExporter(file, **kwargs) - Exports Items in JSON format to the specified file-like object, writing all + Exports items in JSON format to the specified file-like object, writing all objects as a list of objects. The additional ``__init__`` method arguments are passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any @@ -394,7 +400,7 @@ JsonLinesItemExporter .. class:: JsonLinesItemExporter(file, **kwargs) - Exports Items in JSON format to the specified file-like object, writing one + Exports items in JSON format to the specified file-like object, writing one JSON-encoded item per line. The additional ``__init__`` method arguments are passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 9e5968a29..24d69040c 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -298,8 +298,8 @@ Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``. Use FEED_EXPORT_FIELDS option to define fields to export and their order. -When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields -defined in dicts or :class:`~.Item` subclasses a spider is yielding. +When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields +defined in :ref:`item objects <topics-items>` yielded by your spider. If an exporter requires a fixed set of fields (this is the case for :ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index c9194caa1..cd6a6d47e 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -27,15 +27,19 @@ Each item pipeline component is a Python class that must implement the following .. method:: process_item(self, item, spider) - This method is called for every item pipeline component. :meth:`process_item` - must either: return a dict with data, return an :class:`~scrapy.item.Item` - (or any descendant class) object, return a - :class:`~twisted.internet.defer.Deferred` or raise - :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer - processed by further pipeline components. + This method is called for every item pipeline component. - :param item: the item scraped - :type item: :class:`~scrapy.item.Item` object or a dict + `item` is an :ref:`item object <item-types>`, see + :ref:`supporting-item-types`. + + :meth:`process_item` must either: return an :ref:`item object <item-types>`, + return a :class:`~twisted.internet.defer.Deferred` or raise a + :exc:`~scrapy.exceptions.DropItem` exception. + + Dropped items are no longer processed by further pipeline components. + + :param item: the scraped item + :type item: :ref:`item object <item-types>` :param spider: the spider which scraped the item :type spider: :class:`~scrapy.spiders.Spider` object @@ -79,16 +83,17 @@ Let's take a look at the following hypothetical pipeline that adjusts the (``price_excludes_vat`` attribute), and drops those items which don't contain a price:: + from itemadapter import ItemAdapter from scrapy.exceptions import DropItem - class PricePipeline: vat_factor = 1.15 def process_item(self, item, spider): - if item.get('price'): - if item.get('price_excludes_vat'): - item['price'] = item['price'] * self.vat_factor + adapter = ItemAdapter(item) + if adapter.get('price'): + if adapter.get('price_excludes_vat'): + adapter['price'] = adapter['price'] * self.vat_factor return item else: raise DropItem("Missing price in %s" % item) @@ -103,6 +108,8 @@ format:: import json + from itemadapter import ItemAdapter + class JsonWriterPipeline: def open_spider(self, spider): @@ -112,7 +119,7 @@ format:: self.file.close() def process_item(self, item, spider): - line = json.dumps(dict(item)) + "\n" + line = json.dumps(ItemAdapter(item).asdict()) + "\n" self.file.write(line) return item @@ -131,6 +138,7 @@ The main point of this example is to show how to use :meth:`from_crawler` method and how to clean up the resources properly.:: import pymongo + from itemadapter import ItemAdapter class MongoPipeline: @@ -155,7 +163,7 @@ method and how to clean up the resources properly.:: self.client.close() def process_item(self, item, spider): - self.db[self.collection_name].insert_one(dict(item)) + self.db[self.collection_name].insert_one(ItemAdapter(item).asdict()) return item .. _MongoDB: https://www.mongodb.com/ @@ -177,10 +185,11 @@ item. :: - import scrapy import hashlib from urllib.parse import quote + import scrapy + from itemadapter import ItemAdapter class ScreenshotPipeline: """Pipeline that uses Splash to render screenshot of @@ -189,7 +198,8 @@ item. SPLASH_URL = "http://localhost:8050/render.png?url={}" async def process_item(self, item, spider): - encoded_item_url = quote(item["url"]) + adapter = ItemAdapter(item) + encoded_item_url = quote(adapter["url"]) screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url) response = await spider.crawler.engine.download(request, spider) @@ -199,14 +209,14 @@ item. return item # Save screenshot to file, filename will be hash of url. - url = item["url"] + url = adapter["url"] url_hash = hashlib.md5(url.encode("utf8")).hexdigest() filename = "{}.png".format(url_hash) with open(filename, "wb") as f: f.write(response.body) # Store filename in item. - item["screenshot_filename"] = filename + adapter["screenshot_filename"] = filename return item .. _Splash: https://splash.readthedocs.io/en/stable/ @@ -219,6 +229,7 @@ already processed. Let's say that our items have a unique id, but our spider returns multiples items with the same id:: + from itemadapter import ItemAdapter from scrapy.exceptions import DropItem class DuplicatesPipeline: @@ -227,10 +238,11 @@ returns multiples items with the same id:: self.ids_seen = set() def process_item(self, item, spider): - if item['id'] in self.ids_seen: - raise DropItem("Duplicate item found: %s" % item) + adapter = ItemAdapter(item) + if adapter['id'] in self.ids_seen: + raise DropItem("Duplicate item found: %r" % item) else: - self.ids_seen.add(item['id']) + self.ids_seen.add(adapter['id']) return item diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 0941a8a1b..65bf156ac 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -8,29 +8,155 @@ Items :synopsis: Item and Field classes The main goal in scraping is to extract structured data from unstructured -sources, typically, web pages. Scrapy spiders can return the extracted data -as Python dicts. While convenient and familiar, Python dicts lack structure: -it is easy to make a typo in a field name or return inconsistent data, -especially in a larger project with many spiders. +sources, typically, web pages. :ref:`Spiders <topics-spiders>` may return the +extracted data as `items`, Python objects that define key-value pairs. -To define common output data format Scrapy provides the :class:`Item` class. -:class:`Item` objects are simple containers used to collect the scraped data. -They provide an API similar to :class:`dict` API with a convenient syntax -for declaring their available fields. +Scrapy supports :ref:`multiple types of items <item-types>`. When you create an +item, you may use whichever type of item you want. When you write code that +receives an item, your code should :ref:`work for any item type +<supporting-item-types>`. -Various Scrapy components use extra information provided by Items: -exporters look at declared fields to figure out columns to export, -serialization can be customized using Item fields metadata, :mod:`trackref` -tracks Item instances to help find memory leaks -(see :ref:`topics-leaks-trackrefs`), etc. +.. _item-types: + +Item Types +========== + +Scrapy supports the following types of items, via the `itemadapter`_ library: +:ref:`dictionaries <dict-items>`, :ref:`Item objects <item-objects>`, +:ref:`dataclass objects <dataclass-items>`, and :ref:`attrs objects <attrs-items>`. + +.. _itemadapter: https://github.com/scrapy/itemadapter + +.. _dict-items: + +Dictionaries +------------ + +As an item type, :class:`dict` is convenient and familiar. + +.. _item-objects: + +Item objects +------------ + +:class:`Item` provides a :class:`dict`-like API plus additional features that +make it the most feature-complete item type: + +.. class:: Item([arg]) + + :class:`Item` objects replicate the standard :class:`dict` API, including + its ``__init__`` method. + + :class:`Item` allows defining field names, so that: + + - :class:`KeyError` is raised when using undefined field names (i.e. + prevents typos going unnoticed) + + - :ref:`Item exporters <topics-exporters>` can export all fields by + default even if the first scraped object does not have values for all + of them + + :class:`Item` also allows defining field metadata, which can be used to + :ref:`customize serialization <topics-exporters-field-serialization>`. + + :mod:`trackref` tracks :class:`Item` objects to help find memory leaks + (see :ref:`topics-leaks-trackrefs`). + + :class:`Item` objects also provide the following additional API members: + + .. automethod:: copy + + .. automethod:: deepcopy + + .. attribute:: fields + + A dictionary containing *all declared fields* for this Item, not only + those populated. The keys are the field names and the values are the + :class:`Field` objects used in the :ref:`Item declaration + <topics-items-declaring>`. + +Example:: + + from scrapy.item import Item, Field + + class CustomItem(Item): + one_field = Field() + another_field = Field() + +.. _dataclass-items: + +Dataclass objects +----------------- + +.. versionadded:: 2.2 + +:func:`~dataclasses.dataclass` allows defining item classes with field names, +so that :ref:`item exporters <topics-exporters>` can export all fields by +default even if the first scraped object does not have values for all of them. + +Additionally, ``dataclass`` items also allow to: + +* define the type and default value of each defined field. + +* define custom field metadata through :func:`dataclasses.field`, which can be used to + :ref:`customize serialization <topics-exporters-field-serialization>`. + +They work natively in Python 3.7 or later, or using the `dataclasses +backport`_ in Python 3.6. + +.. _dataclasses backport: https://pypi.org/project/dataclasses/ + +Example:: + + from dataclasses import dataclass + + @dataclass + class CustomItem: + one_field: str + another_field: int + +.. note:: Field types are not enforced at run time. + +.. _attrs-items: + +attr.s objects +-------------- + +.. versionadded:: 2.2 + +:func:`attr.s` allows defining item classes with field names, +so that :ref:`item exporters <topics-exporters>` can export all fields by +default even if the first scraped object does not have values for all of them. + +Additionally, ``attr.s`` items also allow to: + +* define the type and default value of each defined field. + +* define custom field :ref:`metadata <attrs:metadata>`, which can be used to + :ref:`customize serialization <topics-exporters-field-serialization>`. + +In order to use this type, the :doc:`attrs package <attrs:index>` needs to be installed. + +Example:: + + import attr + + @attr.s + class CustomItem: + one_field = attr.ib() + another_field = attr.ib() + + +Working with Item objects +========================= .. _topics-items-declaring: -Declaring Items -=============== +Declaring Item subclasses +------------------------- -Items are declared using a simple class definition syntax and :class:`Field` -objects. Here is an example:: +Item subclasses are declared using a simple class definition syntax and +:class:`Field` objects. Here is an example:: import scrapy @@ -48,10 +174,11 @@ objects. Here is an example:: .. _Django: https://www.djangoproject.com/ .. _Django Models: https://docs.djangoproject.com/en/dev/topics/db/models/ + .. _topics-items-fields: -Item Fields -=========== +Declaring fields +---------------- :class:`Field` objects are used to specify metadata for each field. For example, the serializer function for the ``last_updated`` field illustrated in @@ -72,15 +199,31 @@ It's important to note that the :class:`Field` objects used to declare the item do not stay assigned as class attributes. Instead, they can be accessed through the :attr:`Item.fields` attribute. -Working with Items -================== +.. class:: Field([arg]) + + The :class:`Field` class is just an alias to the built-in :class:`dict` class and + doesn't provide any extra functionality or attributes. In other words, + :class:`Field` objects are plain-old Python dicts. A separate class is used + to support the :ref:`item declaration syntax <topics-items-declaring>` + based on class attributes. + +.. note:: Field metadata can also be declared for ``dataclass`` and ``attrs`` + items. Please refer to the documentation for `dataclasses.field`_ and + `attr.ib`_ for additional information. + + .. _dataclasses.field: https://docs.python.org/3/library/dataclasses.html#dataclasses.field + .. _attr.ib: https://www.attrs.org/en/stable/api.html#attr.ib + + +Working with Item objects +------------------------- Here are some examples of common tasks performed with items, using the ``Product`` item :ref:`declared above <topics-items-declaring>`. You will notice the API is very similar to the :class:`dict` API. Creating items --------------- +'''''''''''''' >>> product = Product(name='Desktop PC', price=1000) >>> print(product) @@ -88,7 +231,7 @@ Product(name='Desktop PC', price=1000) Getting field values --------------------- +'''''''''''''''''''' >>> product['name'] Desktop PC @@ -128,7 +271,7 @@ False Setting field values --------------------- +'''''''''''''''''''' >>> product['last_updated'] = 'today' >>> product['last_updated'] @@ -141,7 +284,7 @@ KeyError: 'Product does not support field: lala' Accessing all populated values ------------------------------- +'''''''''''''''''''''''''''''' To access all populated values, just use the typical :class:`dict` API: @@ -155,7 +298,7 @@ To access all populated values, just use the typical :class:`dict` API: .. _copying-items: Copying items -------------- +''''''''''''' To copy an item, you must first decide whether you want a shallow copy or a deep copy. @@ -183,7 +326,7 @@ To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead Other common tasks ------------------- +'''''''''''''''''' Creating dicts from items: @@ -201,8 +344,8 @@ Traceback (most recent call last): KeyError: 'Product does not support field: lala' -Extending Items -=============== +Extending Item subclasses +------------------------- You can extend Items (to add more fields or to change some metadata for some fields) by declaring a subclass of your original Item. @@ -222,39 +365,25 @@ appending more values, or changing existing values, like this:: That adds (or replaces) the ``serializer`` metadata key for the ``name`` field, keeping all the previously existing metadata values. -Item objects -============ -.. class:: Item([arg]) +.. _supporting-item-types: - Return a new Item optionally initialized from the given argument. +Supporting All Item Types +========================= - Items replicate the standard :class:`dict` API, including its ``__init__`` - method, and also provide the following additional API members: +In code that receives an item, such as methods of :ref:`item pipelines +<topics-item-pipeline>` or :ref:`spider middlewares +<topics-spider-middleware>`, it is a good practice to use the +:class:`~itemadapter.ItemAdapter` class and the +:func:`~itemadapter.is_item` function to write code that works for +any :ref:`supported item type <item-types>`: - .. automethod:: copy +.. autoclass:: itemadapter.ItemAdapter - .. automethod:: deepcopy +.. autofunction:: itemadapter.is_item - .. attribute:: fields - A dictionary containing *all declared fields* for this Item, not only - those populated. The keys are the field names and the values are the - :class:`Field` objects used in the :ref:`Item declaration - <topics-items-declaring>`. - -Field objects -============= - -.. class:: Field([arg]) - - The :class:`Field` class is just an alias to the built-in :class:`dict` class and - doesn't provide any extra functionality or attributes. In other words, - :class:`Field` objects are plain-old Python dicts. A separate class is used - to support the :ref:`item declaration syntax <topics-items-declaring>` - based on class attributes. - -Other classes related to Item -============================= +Other classes related to items +============================== .. autoclass:: ItemMeta diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index ceb708c7e..3224241fc 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -4,7 +4,7 @@ Debugging memory leaks ====================== -In Scrapy, objects such as Requests, Responses and Items have a finite +In Scrapy, objects such as requests, responses and items have a finite lifetime: they are created, used for a while, and finally destroyed. From all those objects, the Request is probably the one with the longest @@ -61,8 +61,8 @@ Debugging memory leaks with ``trackref`` ======================================== :mod:`trackref` is a module provided by Scrapy to debug the most common cases of -memory leaks. It basically tracks the references to all live Requests, -Responses, Item and Selector objects. +memory leaks. It basically tracks the references to all live Request, +Response, Item, Spider and Selector objects. You can enter the telnet console and inspect how many objects (of the classes mentioned above) are currently alive using the ``prefs()`` function which is an @@ -200,11 +200,10 @@ Debugging memory leaks with muppy ``trackref`` provides a very convenient mechanism for tracking down memory leaks, but it only keeps track of the objects that are more likely to cause -memory leaks (Requests, Responses, Items, and Selectors). However, there are -other cases where the memory leaks could come from other (more or less obscure) -objects. If this is your case, and you can't find your leaks using ``trackref``, -you still have another resource: the muppy library. - +memory leaks. However, there are other cases where the memory leaks could come +from other (more or less obscure) objects. If this is your case, and you can't +find your leaks using ``trackref``, you still have another resource: the muppy +library. You can use muppy from `Pympler`_. diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index eb804f1db..6645bf123 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -7,13 +7,12 @@ Item Loaders .. module:: scrapy.loader :synopsis: Item Loader class -Item Loaders provide a convenient mechanism for populating scraped :ref:`Items -<topics-items>`. Even though Items can be populated using their own -dictionary-like API, Item Loaders provide a much more convenient API for -populating them from a scraping process, by automating some common tasks like -parsing the raw extracted data before assigning it. +Item Loaders provide a convenient mechanism for populating scraped :ref:`items +<topics-items>`. Even though items can be populated directly, Item Loaders provide a +much more convenient API for populating them from a scraping process, by automating +some common tasks like parsing the raw extracted data before assigning it. -In other words, :ref:`Items <topics-items>` provide the *container* of +In other words, :ref:`items <topics-items>` provide the *container* of scraped data, while Item Loaders provide the mechanism for *populating* that container. @@ -25,10 +24,10 @@ Using Item Loaders to populate items ==================================== To use an Item Loader, you must first instantiate it. You can either -instantiate it with a dict-like object (e.g. Item or dict) or without one, in -which case an Item is automatically instantiated in the Item Loader ``__init__`` method -using the Item class specified in the :attr:`ItemLoader.default_item_class` -attribute. +instantiate it with an :ref:`item object <topics-items>` or without one, in which +case an instance of :class:`~scrapy.item.Item` is automatically created in the +Item Loader ``__init__`` method using the :class:`~scrapy.item.Item` subclass +specified in the :attr:`ItemLoader.default_item_class` attribute. Then, you start collecting values into the Item Loader, typically using :ref:`Selectors <topics-selectors>`. You can add more than one value to @@ -88,7 +87,7 @@ received (through the :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css` :meth:`~ItemLoader.add_value` methods) and the result of the input processor is collected and kept inside the ItemLoader. After collecting all data, the :meth:`ItemLoader.load_item` method is called to populate and get the populated -:class:`~scrapy.item.Item` object. That's when the output processor is +:ref:`item object <topics-items>`. That's when the output processor is called with the data previously collected (and processed using the input processor). The result of the output processor is the final value that gets assigned to the item. @@ -153,12 +152,10 @@ Last, but not least, Scrapy comes with some :ref:`commonly used processors <topics-loaders-available-processors>` built-in for convenience. - Declaring Item Loaders ====================== -Item Loaders are declared like Items, by using a class definition syntax. Here -is an example:: +Item Loaders are declared using a class definition syntax. Here is an example:: from scrapy.loader import ItemLoader from scrapy.loader.processors import TakeFirst, MapCompose, Join @@ -275,9 +272,9 @@ ItemLoader objects .. class:: ItemLoader([item, selector, response], **kwargs) - Return a new Item Loader for populating the given Item. If no item is - given, one is instantiated automatically using the class in - :attr:`default_item_class`. + Return a new Item Loader for populating the given :ref:`item object + <topics-items>`. If no item object is given, one is instantiated + automatically using the class in :attr:`default_item_class`. When instantiated with a ``selector`` or a ``response`` parameters the :class:`ItemLoader` class provides convenient mechanisms for extracting @@ -286,7 +283,7 @@ ItemLoader objects :param item: The item instance to populate using subsequent calls to :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`, or :meth:`~ItemLoader.add_value`. - :type item: :class:`~scrapy.item.Item` object + :type item: :ref:`item object <topics-items>` :param selector: The selector to extract data from, when using the :meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath` @@ -444,17 +441,19 @@ ItemLoader objects Create a nested loader with an xpath selector. The supplied selector is applied relative to selector associated - with this :class:`ItemLoader`. The nested loader shares the :class:`Item` - with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`, - :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected. + with this :class:`ItemLoader`. The nested loader shares the :ref:`item + object <topics-items>` with the parent :class:`ItemLoader` so calls to + :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will + behave as expected. .. method:: nested_css(css) Create a nested loader with a css selector. The supplied selector is applied relative to selector associated - with this :class:`ItemLoader`. The nested loader shares the :class:`Item` - with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`, - :meth:`add_value`, :meth:`replace_value`, etc. will behave as expected. + with this :class:`ItemLoader`. The nested loader shares the :ref:`item + object <topics-items>` with the parent :class:`ItemLoader` so calls to + :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will + behave as expected. .. method:: get_collected_values(field_name) @@ -477,7 +476,7 @@ ItemLoader objects .. attribute:: item - The :class:`~scrapy.item.Item` object being parsed by this Item Loader. + The :ref:`item object <topics-items>` being parsed by this Item Loader. This is mostly used as a property so when attempting to override this value, you may want to check out :attr:`default_item_class` first. @@ -488,8 +487,8 @@ ItemLoader objects .. attribute:: default_item_class - An Item class (or factory), used to instantiate items when not given in - the ``__init__`` method. + An :ref:`item object <topics-items>` class or factory, used to + instantiate items when not given in the ``__init__`` method. .. attribute:: default_input_processor diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 86550d7a4..01de3dedb 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -156,7 +156,7 @@ following forms:: ftp://username:password@address:port/path ftp://address:port/path - + If ``username`` and ``password`` are not provided, they are taken from the :setting:`FTP_USER` and :setting:`FTP_PASSWORD` settings respectively. @@ -243,20 +243,22 @@ Usage example .. setting:: IMAGES_URLS_FIELD .. setting:: IMAGES_RESULT_FIELD -In order to use a media pipeline first, :ref:`enable it +In order to use a media pipeline, first :ref:`enable it <topics-media-pipeline-enabling>`. -Then, if a spider returns a dict with the URLs key (``file_urls`` or -``image_urls``, for the Files or Images Pipeline respectively), the pipeline will -put the results under respective key (``files`` or ``images``). +Then, if a spider returns an :ref:`item object <topics-items>` with the URLs +field (``file_urls`` or ``image_urls``, for the Files or Images Pipeline +respectively), the pipeline will put the results under the respective field +(``files`` or ``images``). -If you prefer to use :class:`~.Item`, then define a custom item with the -necessary fields, like in this example for Images Pipeline:: +When using :ref:`item types <item-types>` for which fields are defined beforehand, +you must define both the URLs field and the results field. For example, when +using the images pipeline, items must define both the ``image_urls`` and the +``images`` field. For instance, using the :class:`~scrapy.item.Item` class:: import scrapy class MyItem(scrapy.Item): - # ... other item fields ... image_urls = scrapy.Field() images = scrapy.Field() @@ -445,8 +447,11 @@ See here the methods that you can override in your custom Files Pipeline: :meth:`~get_media_requests` method and return a Request for each file URL:: + from itemadapter import ItemAdapter + def get_media_requests(self, item, info): - for file_url in item['file_urls']: + adapter = ItemAdapter(item) + for file_url in adapter['file_urls']: yield scrapy.Request(file_url) Those requests will be processed by the pipeline and, when they have finished @@ -509,13 +514,15 @@ See here the methods that you can override in your custom Files Pipeline: store the downloaded file paths (passed in results) in the ``file_paths`` item field, and we drop the item if it doesn't contain any files:: + from itemadapter import ItemAdapter from scrapy.exceptions import DropItem def item_completed(self, results, item, info): file_paths = [x['path'] for ok, x in results if ok] if not file_paths: raise DropItem("Item contains no files") - item['file_paths'] = file_paths + adapter = ItemAdapter(item) + adapter['file_paths'] = file_paths return item By default, the :meth:`item_completed` method returns the item. @@ -589,8 +596,9 @@ Here is a full example of the Images Pipeline whose methods are exemplified above:: import scrapy - from scrapy.pipelines.images import ImagesPipeline + from itemadapter import ItemAdapter from scrapy.exceptions import DropItem + from scrapy.pipelines.images import ImagesPipeline class MyImagesPipeline(ImagesPipeline): @@ -602,7 +610,8 @@ above:: image_paths = [x['path'] for ok, x in results if ok] if not image_paths: raise DropItem("Item contains no images") - item['image_paths'] = image_paths + adapter = ItemAdapter(item) + adapter['image_paths'] = image_paths return item diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f06d9db3c..5178f272f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -236,8 +236,8 @@ CONCURRENT_ITEMS Default: ``100`` -Maximum number of concurrent items (per response) to process in parallel in the -Item Processor (also known as the :ref:`Item Pipeline <topics-item-pipeline>`). +Maximum number of concurrent items (per response) to process in parallel in +:ref:`item pipelines <topics-item-pipeline>`. .. setting:: CONCURRENT_REQUESTS diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index fe4fb0834..255ba9d3f 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -151,8 +151,8 @@ item_scraped This signal supports returning deferreds from its handlers. - :param item: the item scraped - :type item: dict or :class:`~scrapy.item.Item` object + :param item: the scraped item + :type item: :ref:`item object <item-types>` :param spider: the spider which scraped the item :type spider: :class:`~scrapy.spiders.Spider` object @@ -172,7 +172,7 @@ item_dropped This signal supports returning deferreds from its handlers. :param item: the item dropped from the :ref:`topics-item-pipeline` - :type item: dict or :class:`~scrapy.item.Item` object + :type item: :ref:`item object <item-types>` :param spider: the spider which scraped the item :type spider: :class:`~scrapy.spiders.Spider` object @@ -196,8 +196,8 @@ item_error This signal supports returning deferreds from its handlers. - :param item: the item dropped from the :ref:`topics-item-pipeline` - :type item: dict or :class:`~scrapy.item.Item` object + :param item: the item that caused the error in the :ref:`topics-item-pipeline` + :type item: :ref:`item object <item-types>` :param response: the response being processed when the exception was raised :type response: :class:`~scrapy.http.Response` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index d49a2209d..c6cbdba76 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -102,29 +102,28 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`. it has processed the response. :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` - objects. + :class:`~scrapy.http.Request` objects and :ref:`item object + <topics-items>`. :param response: the response which generated this output from the spider :type response: :class:`~scrapy.http.Response` object :param result: the result returned by the spider - :type result: an iterable of :class:`~scrapy.http.Request`, dict - or :class:`~scrapy.item.Item` objects + :type result: an iterable of :class:`~scrapy.http.Request` objects and + :ref:`item object <topics-items>` :param spider: the spider whose result is being processed :type spider: :class:`~scrapy.spiders.Spider` object - .. method:: process_spider_exception(response, exception, spider) This method is called when a spider or :meth:`process_spider_output` method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Request`, dict or - :class:`~scrapy.item.Item` objects. + iterable of :class:`~scrapy.http.Request` objects and :ref:`item object + <topics-items>`. If it returns ``None``, Scrapy will continue processing this exception, executing any other :meth:`process_spider_exception` in the following diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 8ff5e7292..d4d6e2ea0 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -23,8 +23,8 @@ For spiders, the scraping cycle goes through something like this: :attr:`~scrapy.spiders.Spider.parse` method as callback function for the Requests. -2. In the callback function, you parse the response (web page) and return either - dicts with extracted data, :class:`~scrapy.item.Item` objects, +2. In the callback function, you parse the response (web page) and return + :ref:`item objects <topics-items>`, :class:`~scrapy.http.Request` objects, or an iterable of these objects. Those Requests will also contain a callback (maybe the same) and will then be downloaded by Scrapy and then their @@ -179,8 +179,8 @@ scrapy.Spider the same requirements as the :class:`Spider` class. This method, as well as any other Request callback, must return an - iterable of :class:`~scrapy.http.Request` and/or - dicts or :class:`~scrapy.item.Item` objects. + iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects + <topics-items>`. :param response: the response to parse :type response: :class:`~scrapy.http.Response` @@ -234,7 +234,7 @@ Return multiple Requests and items from a single callback:: yield scrapy.Request(response.urljoin(href), self.parse) Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; -to give data more structure you can use :ref:`topics-items`:: +to give data more structure you can use :class:`~scrapy.item.Item` objects:: import scrapy from myproject.items import MyItem @@ -364,7 +364,7 @@ CrawlSpider This method is called for the start_urls responses. It allows to parse the initial responses and must return either an - :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request` + :ref:`item object <topics-items>`, a :class:`~scrapy.http.Request` object, or an iterable containing any of them. Crawling rules @@ -383,7 +383,7 @@ Crawling rules object with that name will be used) to be called for each link extracted with the specified link extractor. This callback receives a :class:`~scrapy.http.Response` as its first argument and must return either a single instance or an iterable of - :class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects + :ref:`item objects <topics-items>` and/or :class:`~scrapy.http.Request` objects (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response` object will contain the text of the link that produced the :class:`~scrapy.http.Request` in its ``meta`` dictionary (under the ``link_text`` key) @@ -531,7 +531,7 @@ XMLFeedSpider (``itertag``). Receives the response and an :class:`~scrapy.selector.Selector` for each node. Overriding this method is mandatory. Otherwise, you spider won't work. This method - must return either a :class:`~scrapy.item.Item` object, a + must return an :ref:`item object <topics-items>`, a :class:`~scrapy.http.Request` object, or an iterable containing any of them. @@ -541,7 +541,7 @@ XMLFeedSpider spider, and it's intended to perform any last time processing required before returning the results to the framework core, for example setting the item IDs. It receives a list of results and the response which originated - those results. It must return a list of results (Items or Requests). + those results. It must return a list of results (items or requests). XMLFeedSpider example diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 580fd2828..8b7fa8b58 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,11 +1,11 @@ import json import logging +from itemadapter import is_item, ItemAdapter from w3lib.url import is_url from scrapy.commands import ScrapyCommand from scrapy.http import Request -from scrapy.item import _BaseItem from scrapy.utils import display from scrapy.utils.conf import arglist_to_dict from scrapy.utils.spider import iterate_spider_output, spidercls_for_request @@ -81,7 +81,7 @@ class Command(ScrapyCommand): items = self.items.get(lvl, []) print("# Scraped Items ", "-" * 60) - display.pprint([dict(x) for x in items], colorize=colour) + display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour) def print_requests(self, lvl=None, colour=True): if lvl is None: @@ -117,7 +117,7 @@ class Command(ScrapyCommand): items, requests = [], [] for x in iterate_spider_output(callback(response, **cb_kwargs)): - if isinstance(x, (_BaseItem, dict)): + if is_item(x): items.append(x) elif isinstance(x, Request): requests.append(x) diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index cdc2bac15..34f0d36d4 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,10 +1,10 @@ import json -from scrapy.item import _BaseItem -from scrapy.http import Request -from scrapy.exceptions import ContractFail +from itemadapter import is_item, ItemAdapter from scrapy.contracts import Contract +from scrapy.exceptions import ContractFail +from scrapy.http import Request # contracts @@ -48,11 +48,11 @@ class ReturnsContract(Contract): """ name = 'returns' - objects = { - 'request': Request, - 'requests': Request, - 'item': (_BaseItem, dict), - 'items': (_BaseItem, dict), + object_type_verifiers = { + 'request': lambda x: isinstance(x, Request), + 'requests': lambda x: isinstance(x, Request), + 'item': is_item, + 'items': is_item, } def __init__(self, *args, **kwargs): @@ -64,7 +64,7 @@ class ReturnsContract(Contract): % len(self.args) ) self.obj_name = self.args[0] or None - self.obj_type = self.objects[self.obj_name] + self.obj_type_verifier = self.object_type_verifiers[self.obj_name] try: self.min_bound = int(self.args[1]) @@ -79,7 +79,7 @@ class ReturnsContract(Contract): def post_process(self, output): occurrences = 0 for x in output: - if isinstance(x, self.obj_type): + if self.obj_type_verifier(x): occurrences += 1 assertion = (self.min_bound <= occurrences <= self.max_bound) @@ -103,8 +103,8 @@ class ScrapesContract(Contract): def post_process(self, output): for x in output: - if isinstance(x, (_BaseItem, dict)): - missing = [arg for arg in self.args if arg not in x] + if is_item(x): + missing = [arg for arg in self.args if arg not in ItemAdapter(x)] if missing: - raise ContractFail( - "Missing fields: %s" % ", ".join(missing)) + missing_str = ", ".join(missing) + raise ContractFail("Missing fields: %s" % missing_str) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 6785e103d..d07c7aa62 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -4,18 +4,18 @@ extracts information from them""" import logging from collections import deque -from twisted.python.failure import Failure +from itemadapter import is_item from twisted.internet import defer +from twisted.python.failure import Failure -from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback -from scrapy.utils.spider import iterate_spider_output -from scrapy.utils.misc import load_object, warn_on_generator_with_return_value -from scrapy.utils.log import logformatter_adapter, failure_to_exc_info -from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy import signals -from scrapy.http import Request, Response -from scrapy.item import _BaseItem from scrapy.core.spidermw import SpiderMiddlewareManager +from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest +from scrapy.http import Request, Response +from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel +from scrapy.utils.log import failure_to_exc_info, logformatter_adapter +from scrapy.utils.misc import load_object, warn_on_generator_with_return_value +from scrapy.utils.spider import iterate_spider_output logger = logging.getLogger(__name__) @@ -191,7 +191,7 @@ class Scraper: """ if isinstance(output, Request): self.crawler.engine.crawl(request=output, spider=spider) - elif isinstance(output, (_BaseItem, dict)): + elif is_item(output): self.slot.itemproc_size += 1 dfd = self.itemproc.process_item(output, spider) dfd.addBoth(self._itemproc_finished, output, response, spider) @@ -200,10 +200,11 @@ class Scraper: pass else: typename = type(output).__name__ - logger.error('Spider must return Request, BaseItem, dict or None, ' - 'got %(typename)r in %(request)s', - {'request': request, 'typename': typename}, - extra={'spider': spider}) + logger.error( + 'Spider must return request, item, or None, got %(typename)r in %(request)s', + {'request': request, 'typename': typename}, + extra={'spider': spider}, + ) def _log_download_errors(self, spider_failure, download_failure, request, spider): """Log and silence errors that come from the engine (typically download diff --git a/scrapy/exporters.py b/scrapy/exporters.py index de009082a..712572673 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -4,16 +4,18 @@ Item Exporters are used to export/serialize items into different formats. import csv import io -import pprint import marshal -import warnings import pickle +import pprint +import warnings from xml.sax.saxutils import XMLGenerator -from scrapy.utils.serialize import ScrapyJSONEncoder -from scrapy.utils.python import to_bytes, to_unicode, is_listlike -from scrapy.item import _BaseItem +from itemadapter import is_item, ItemAdapter + from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.item import _BaseItem +from scrapy.utils.python import is_listlike, to_bytes, to_unicode +from scrapy.utils.serialize import ScrapyJSONEncoder __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', @@ -56,11 +58,14 @@ class BaseItemExporter: """Return the fields to export as an iterable of tuples (name, serialized_value) """ + item = ItemAdapter(item) + if include_empty is None: include_empty = self.export_empty_fields + if self.fields_to_export is None: - if include_empty and not isinstance(item, dict): - field_iter = item.fields.keys() + if include_empty: + field_iter = item.field_names() else: field_iter = item.keys() else: @@ -71,8 +76,8 @@ class BaseItemExporter: for field_name in field_iter: if field_name in item: - field = {} if isinstance(item, dict) else item.fields[field_name] - value = self.serialize_field(field, field_name, item[field_name]) + field_meta = item.get_field_meta(field_name) + value = self.serialize_field(field_meta, field_name, item[field_name]) else: value = default_value @@ -297,6 +302,7 @@ class PythonItemExporter(BaseItemExporter): .. _msgpack: https://pypi.org/project/msgpack/ """ + def _configure(self, options, dont_fail=False): self.binary = options.pop('binary', True) super(PythonItemExporter, self)._configure(options, dont_fail) @@ -314,22 +320,22 @@ class PythonItemExporter(BaseItemExporter): def _serialize_value(self, value): if isinstance(value, _BaseItem): return self.export_item(value) - if isinstance(value, dict): - return dict(self._serialize_dict(value)) - if is_listlike(value): + elif is_item(value): + return dict(self._serialize_item(value)) + elif is_listlike(value): return [self._serialize_value(v) for v in value] encode_func = to_bytes if self.binary else to_unicode if isinstance(value, (str, bytes)): return encode_func(value, encoding=self.encoding) return value - def _serialize_dict(self, value): - for key, val in value.items(): + def _serialize_item(self, item): + for key, value in ItemAdapter(item).items(): key = to_bytes(key) if self.binary else key - yield key, self._serialize_value(val) + yield key, self._serialize_value(value) def export_item(self, item): result = dict(self._get_serialized_fields(item)) if self.binary: - result = dict(self._serialize_dict(result)) + result = dict(self._serialize_item(result)) return result diff --git a/scrapy/item.py b/scrapy/item.py index 97dfed976..4ab83d1a0 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -36,7 +36,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta): """ def __new__(cls, *args, **kwargs): - if issubclass(cls, BaseItem) and not (issubclass(cls, Item) or issubclass(cls, DictItem)): + if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)): warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', ScrapyDeprecationWarning, stacklevel=2) return super(BaseItem, cls).__new__(cls, *args, **kwargs) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 21c4fb376..18f57945f 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -6,6 +6,8 @@ See documentation in docs/topics/loaders.rst from collections import defaultdict from contextlib import suppress +from itemadapter import ItemAdapter + from scrapy.item import Item from scrapy.loader.common import wrap_loader_context from scrapy.loader.processors import Identity @@ -44,7 +46,7 @@ class ItemLoader: self._local_item = context['item'] = item self._local_values = defaultdict(list) # values from initial item - for field_name, value in item.items(): + for field_name, value in ItemAdapter(item).items(): self._values[field_name] += arg_to_iter(value) @property @@ -127,13 +129,12 @@ class ItemLoader: return value def load_item(self): - item = self.item + adapter = ItemAdapter(self.item) for field_name in tuple(self._values): value = self.get_output_value(field_name) if value is not None: - item[field_name] = value - - return item + adapter[field_name] = value + return adapter.item def get_output_value(self, field_name): proc = self.get_output_processor(field_name) @@ -174,11 +175,8 @@ class ItemLoader: value, type(e).__name__, str(e))) def _get_item_field_attr(self, field_name, key, default=None): - if isinstance(self.item, Item): - value = self.item.fields[field_name].get(key, default) - else: - value = default - return value + field_meta = ItemAdapter(self.item).get_field_meta(field_name) + return field_meta.get(key, default) def _check_selector_method(self): if self.selector is None: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 7d86d0d56..487382a38 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -10,24 +10,26 @@ import mimetypes import os import time from collections import defaultdict -from email.utils import parsedate_tz, mktime_tz +from contextlib import suppress +from email.utils import mktime_tz, parsedate_tz from ftplib import FTP from io import BytesIO from urllib.parse import urlparse +from itemadapter import ItemAdapter from twisted.internet import defer, threads +from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.http import Request from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings -from scrapy.exceptions import NotConfigured, IgnoreRequest -from scrapy.http import Request -from scrapy.utils.misc import md5sum -from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.python import to_bytes -from scrapy.utils.request import referer_str from scrapy.utils.boto import is_botocore from scrapy.utils.datatypes import CaselessDict from scrapy.utils.ftp import ftp_store_file +from scrapy.utils.log import failure_to_exc_info +from scrapy.utils.misc import md5sum +from scrapy.utils.python import to_bytes +from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) @@ -517,7 +519,8 @@ class FilesPipeline(MediaPipeline): # Overridable Interface def get_media_requests(self, item, info): - return [Request(x) for x in item.get(self.files_urls_field, [])] + urls = ItemAdapter(item).get(self.files_urls_field, []) + return [Request(u) for u in urls] def file_downloaded(self, response, request, info): path = self.file_path(request, response=response, info=info) @@ -528,8 +531,8 @@ class FilesPipeline(MediaPipeline): return checksum def item_completed(self, results, item, info): - if isinstance(item, dict) or self.files_result_field in item.fields: - item[self.files_result_field] = [x for ok, x in results if ok] + with suppress(KeyError): + ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] return item def file_path(self, request, response=None, info=None): diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index aeb520442..46f2bfb58 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -5,17 +5,19 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib +from contextlib import suppress from io import BytesIO +from itemadapter import ItemAdapter from PIL import Image +from scrapy.exceptions import DropItem +from scrapy.http import Request +from scrapy.pipelines.files import FileException, FilesPipeline +# TODO: from scrapy.pipelines.media import MediaPipeline +from scrapy.settings import Settings from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes -from scrapy.http import Request -from scrapy.settings import Settings -from scrapy.exceptions import DropItem -# TODO: from scrapy.pipelines.media import MediaPipeline -from scrapy.pipelines.files import FileException, FilesPipeline class NoimagesDrop(DropItem): @@ -157,11 +159,12 @@ class ImagesPipeline(FilesPipeline): return image, buf def get_media_requests(self, item, info): - return [Request(x) for x in item.get(self.images_urls_field, [])] + urls = ItemAdapter(item).get(self.images_urls_field, []) + return [Request(u) for u in urls] def item_completed(self, results, item, info): - if isinstance(item, dict) or self.images_result_field in item.fields: - item[self.images_result_field] = [x for ok, x in results if ok] + with suppress(KeyError): + ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] return item def file_path(self, request, response=None, info=None): diff --git a/scrapy/shell.py b/scrapy/shell.py index 3ff5a8ad8..10de119ce 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst import os import signal +from itemadapter import is_item from twisted.internet import threads, defer from twisted.python import threadable from w3lib.url import any_to_uri @@ -13,20 +14,18 @@ from w3lib.url import any_to_uri from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response -from scrapy.item import _BaseItem from scrapy.settings import Settings from scrapy.spiders import Spider -from scrapy.utils.console import start_python_console +from scrapy.utils.conf import get_config +from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser -from scrapy.utils.conf import get_config -from scrapy.utils.console import DEFAULT_PYTHON_SHELLS class Shell: - relevant_classes = (Crawler, Spider, Request, Response, _BaseItem, Settings) + relevant_classes = (Crawler, Spider, Request, Response, Settings) def __init__(self, crawler, update_vars=None, code=None): self.crawler = crawler @@ -154,7 +153,7 @@ class Shell: return "\n".join("[s] %s" % line for line in b) def _is_relevant(self, value): - return isinstance(value, self.relevant_classes) + return isinstance(value, self.relevant_classes) or is_item(value) def inspect_response(response, spider): diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index a4ff8010d..5aad7398a 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -31,7 +31,7 @@ class XMLFeedSpider(Spider): processing required before returning the results to the framework core, for example setting the item GUIDs. It receives a list of results and the response which originated that results. It must return a list of - results (Items or Requests). + results (items or requests). """ return results diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 6490f52a7..bd09890fe 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -5,6 +5,9 @@ from scrapy import signals +# useful for handling different item types with a single interface +from itemadapter import is_item, ItemAdapter + class ${ProjectName}SpiderMiddleware: # Not all methods need to be defined. If a method is not defined, @@ -29,7 +32,7 @@ class ${ProjectName}SpiderMiddleware: # Called with the results returned from the Spider, after # it has processed the response. - # Must return an iterable of Request, dict or Item objects. + # Must return an iterable of Request, or item objects. for i in result: yield i @@ -37,8 +40,7 @@ class ${ProjectName}SpiderMiddleware: # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. - # Should return either None or an iterable of Request, dict - # or Item objects. + # Should return either None or an iterable of Request or item objects. pass def process_start_requests(self, start_requests, spider): diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index ce0edd335..e845f43e9 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -4,6 +4,10 @@ # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html +# useful for handling different item types with a single interface +from itemadapter import ItemAdapter + + class ${ProjectName}Pipeline: def process_item(self, item, spider): return item diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index bf73dfa18..dc9604578 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -2,10 +2,10 @@ import json import datetime import decimal +from itemadapter import is_item, ItemAdapter from twisted.internet import defer from scrapy.http import Request, Response -from scrapy.item import _BaseItem class ScrapyJSONEncoder(json.JSONEncoder): @@ -26,8 +26,8 @@ class ScrapyJSONEncoder(json.JSONEncoder): return str(o) elif isinstance(o, defer.Deferred): return str(o) - elif isinstance(o, _BaseItem): - return dict(o) + elif is_item(o): + return ItemAdapter(o).asdict() elif isinstance(o, Request): return "<%s %s %s>" % (type(o).__name__, o.method, o.url) elif isinstance(o, Response): diff --git a/setup.py b/setup.py index 71dc3232d..5a99fd1bf 100644 --- a/setup.py +++ b/setup.py @@ -80,6 +80,7 @@ setup( 'w3lib>=1.17.0', 'zope.interface>=4.1.3', 'protego>=0.1.15', + 'itemadapter>=0.1.0', ], extras_require=extras_require, ) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 91fa1c5b5..dacb86e56 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,4 +1,6 @@ # Tests requirements +attrs +dataclasses; python_version == '3.6' jmespath mitmproxy; python_version >= '3.6' mitmproxy<4.0.0; python_version < '3.6' diff --git a/tests/test_engine.py b/tests/test_engine.py index 6696ee52e..1b848ac72 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -16,9 +16,11 @@ import sys from collections import defaultdict from urllib.parse import urlparse +import attr +from itemadapter import ItemAdapter from pydispatch import dispatcher from testfixtures import LogCapture -from twisted.internet import reactor, defer +from twisted.internet import defer, reactor from twisted.trial import unittest from twisted.web import server, static, util @@ -32,7 +34,7 @@ from scrapy.spiders import Spider from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler -from tests import tests_datadir, get_testdata +from tests import get_testdata, tests_datadir class TestItem(Item): @@ -41,6 +43,13 @@ class TestItem(Item): price = Field() +@attr.s +class AttrsItem: + name = attr.ib(default="") + url = attr.ib(default="") + price = attr.ib(default=0) + + class TestSpider(Spider): name = "scrapytest.org" allowed_domains = ["scrapytest.org", "localhost"] @@ -79,6 +88,27 @@ class DictItemsSpider(TestSpider): item_cls = dict +class AttrsItemsSpider(TestSpider): + item_class = AttrsItem + + +try: + from dataclasses import make_dataclass +except ImportError: + DataClassItemsSpider = None +else: + TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) + + class DataClassItemsSpider(DictItemsSpider): + def parse_item(self, response): + item = super().parse_item(response) + return TestDataClass( + name=item.get('name'), + url=item.get('url'), + price=item.get('price'), + ) + + class ItemZeroDivisionErrorSpider(TestSpider): custom_settings = { "ITEM_PIPELINES": { @@ -204,7 +234,10 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_crawler(self): - for spider in TestSpider, DictItemsSpider: + + for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + if spider is None: + continue self.run = CrawlerRun(spider) yield self.run.run() self._assert_visited_urls() @@ -281,6 +314,7 @@ class EngineTest(unittest.TestCase): def _assert_scraped_items(self): self.assertEqual(2, len(self.run.itemresp)) for item, response in self.run.itemresp: + item = ItemAdapter(item) self.assertEqual(item['url'], response.url) if 'item1.html' in item['url']: self.assertEqual('Item 1 name', item['name']) diff --git a/tests/test_loader.py b/tests/test_loader.py index f14714c75..8a9c6fca9 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,6 +1,9 @@ from functools import partial import unittest +import attr +from itemadapter import ItemAdapter + from scrapy.http import HtmlResponse from scrapy.item import Item, Field from scrapy.loader import ItemLoader @@ -9,6 +12,13 @@ from scrapy.loader.processors import (Compose, Identity, Join, from scrapy.selector import Selector +try: + from dataclasses import make_dataclass, field as dataclass_field +except ImportError: + make_dataclass = None + dataclass_field = None + + # test items class NameItem(Item): name = Field() @@ -28,6 +38,11 @@ class TestNestedItem(Item): image = Field() +@attr.s +class AttrsNameItem: + name = attr.ib(default="") + + # test item loaders class NameItemLoader(ItemLoader): default_item_class = TestItem @@ -466,7 +481,7 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {'name': ['foo']}) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo']}) def test_keep_list(self): """Loaded item should contain values from the initial item""" @@ -474,7 +489,7 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']}) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']}) def test_add_value_singlevalue_singlevalue(self): """Values added after initialization should be appended""" @@ -483,7 +498,7 @@ class InitializationTestMixin: il.add_value('name', 'bar') loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']}) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']}) def test_add_value_singlevalue_list(self): """Values added after initialization should be appended""" @@ -492,7 +507,7 @@ class InitializationTestMixin: il.add_value('name', ['item', 'loader']) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']}) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'item', 'loader']}) def test_add_value_list_singlevalue(self): """Values added after initialization should be appended""" @@ -501,7 +516,7 @@ class InitializationTestMixin: il.add_value('name', 'qwerty') loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']}) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'qwerty']}) def test_add_value_list_list(self): """Values added after initialization should be appended""" @@ -510,7 +525,7 @@ class InitializationTestMixin: il.add_value('name', ['item', 'loader']) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']}) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'item', 'loader']}) def test_get_output_value_singlevalue(self): """Getting output value must not remove value from item""" @@ -519,7 +534,7 @@ class InitializationTestMixin: self.assertEqual(il.get_output_value('name'), ['foo']) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, dict({'name': ['foo']})) + self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo']})) def test_get_output_value_list(self): """Getting output value must not remove value from item""" @@ -528,7 +543,7 @@ class InitializationTestMixin: self.assertEqual(il.get_output_value('name'), ['foo', 'bar']) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']})) + self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo', 'bar']})) def test_values_single(self): """Values from initial item must be added to loader._values""" @@ -551,6 +566,22 @@ class InitializationFromItemTest(InitializationTestMixin, unittest.TestCase): item_class = NameItem +class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase): + item_class = AttrsNameItem + + +@unittest.skipIf(not make_dataclass, "dataclasses module is not available") +class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if make_dataclass: + self.item_class = make_dataclass( + "TestDataClass", + [("name", list, dataclass_field(default_factory=list))], + ) + + class BaseNoInputReprocessingLoader(ItemLoader): title_in = MapCompose(str.upper) title_out = TakeFirst() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 6bbcbc2e9..a023dfcc8 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -2,22 +2,41 @@ import os import random import time from io import BytesIO -from tempfile import mkdtemp from shutil import rmtree -from unittest import mock +from tempfile import mkdtemp +from unittest import mock, skipIf from urllib.parse import urlparse -from twisted.trial import unittest +import attr +from itemadapter import ItemAdapter from twisted.internet import defer +from twisted.trial import unittest -from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore, FTPFilesStore -from scrapy.item import Item, Field from scrapy.http import Request, Response +from scrapy.item import Field, Item +from scrapy.pipelines.files import ( + FilesPipeline, + FSFilesStore, + FTPFilesStore, + GCSFilesStore, + S3FilesStore, +) from scrapy.settings import Settings -from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete -from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete -from scrapy.utils.test import get_ftp_content_and_delete from scrapy.utils.boto import is_botocore +from scrapy.utils.test import ( + assert_aws_environ, + assert_gcs_environ, + get_ftp_content_and_delete, + get_gcs_content_and_delete, + get_s3_content_and_delete, +) + + +try: + from dataclasses import make_dataclass, field as dataclass_field +except ImportError: + make_dataclass = None + dataclass_field = None def _mocked_download_func(request, info): @@ -143,43 +162,88 @@ class FilesPipelineTestCase(unittest.TestCase): p.stop() -class FilesPipelineTestCaseFields(unittest.TestCase): +class FilesPipelineTestCaseFieldsMixin: def test_item_fields_default(self): - class TestItem(Item): - name = Field() - file_urls = Field() - files = Field() - - for cls in TestItem, dict: - url = 'http://www.example.com/files/1.txt' - item = cls({'name': 'item1', 'file_urls': [url]}) - pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'})) - requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) - results = [(True, {'url': url})] - pipeline.item_completed(results, item, None) - self.assertEqual(item['files'], [results[0][1]]) + url = 'http://www.example.com/files/1.txt' + item = self.item_class(name='item1', file_urls=[url]) + pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'})) + requests = list(pipeline.get_media_requests(item, None)) + self.assertEqual(requests[0].url, url) + results = [(True, {'url': url})] + item = pipeline.item_completed(results, item, None) + files = ItemAdapter(item).get("files") + self.assertEqual(files, [results[0][1]]) + self.assertIsInstance(item, self.item_class) def test_item_fields_override_settings(self): - class TestItem(Item): - name = Field() - files = Field() - stored_file = Field() + url = 'http://www.example.com/files/1.txt' + item = self.item_class(name='item1', custom_file_urls=[url]) + pipeline = FilesPipeline.from_settings(Settings({ + 'FILES_STORE': 's3://example/files/', + 'FILES_URLS_FIELD': 'custom_file_urls', + 'FILES_RESULT_FIELD': 'custom_files' + })) + requests = list(pipeline.get_media_requests(item, None)) + self.assertEqual(requests[0].url, url) + results = [(True, {'url': url})] + item = pipeline.item_completed(results, item, None) + custom_files = ItemAdapter(item).get("custom_files") + self.assertEqual(custom_files, [results[0][1]]) + self.assertIsInstance(item, self.item_class) - for cls in TestItem, dict: - url = 'http://www.example.com/files/1.txt' - item = cls({'name': 'item1', 'files': [url]}) - pipeline = FilesPipeline.from_settings(Settings({ - 'FILES_STORE': 's3://example/files/', - 'FILES_URLS_FIELD': 'files', - 'FILES_RESULT_FIELD': 'stored_file' - })) - requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) - results = [(True, {'url': url})] - pipeline.item_completed(results, item, None) - self.assertEqual(item['stored_file'], [results[0][1]]) + +class FilesPipelineTestCaseFieldsDict(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = dict + + +class FilesPipelineTestItem(Item): + name = Field() + # default fields + file_urls = Field() + files = Field() + # overridden fields + custom_file_urls = Field() + custom_files = Field() + + +class FilesPipelineTestCaseFieldsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = FilesPipelineTestItem + + +@skipIf(not make_dataclass, "dataclasses module is not available") +class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if make_dataclass: + self.item_class = make_dataclass( + "FilesPipelineTestDataClass", + [ + ("name", str), + # default fields + ("file_urls", list, dataclass_field(default_factory=list)), + ("files", list, dataclass_field(default_factory=list)), + # overridden fields + ("custom_file_urls", list, dataclass_field(default_factory=list)), + ("custom_files", list, dataclass_field(default_factory=list)), + ], + ) + + +@attr.s +class FilesPipelineTestAttrsItem: + name = attr.ib(default="") + # default fields + file_urls = attr.ib(default=lambda: []) + files = attr.ib(default=lambda: []) + # overridden fields + custom_file_urls = attr.ib(default=lambda: []) + custom_files = attr.ib(default=lambda: []) + + +class FilesPipelineTestCaseFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = FilesPipelineTestAttrsItem class FilesPipelineTestCaseCustomSettings(unittest.TestCase): diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 8ef27fce7..082e9ee21 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -1,17 +1,28 @@ -import io import hashlib +import io import random -from tempfile import mkdtemp from shutil import rmtree +from tempfile import mkdtemp +from unittest import skipIf +import attr +from itemadapter import ItemAdapter from twisted.trial import unittest -from scrapy.item import Item, Field from scrapy.http import Request, Response -from scrapy.settings import Settings +from scrapy.item import Field, Item from scrapy.pipelines.images import ImagesPipeline +from scrapy.settings import Settings from scrapy.utils.python import to_bytes + +try: + from dataclasses import make_dataclass, field as dataclass_field +except ImportError: + make_dataclass = None + dataclass_field = None + + skip = False try: from PIL import Image @@ -124,43 +135,89 @@ class DeprecatedImagesPipeline(ImagesPipeline): return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid) -class ImagesPipelineTestCaseFields(unittest.TestCase): +class ImagesPipelineTestCaseFieldsMixin: def test_item_fields_default(self): - class TestItem(Item): - name = Field() - image_urls = Field() - images = Field() - - for cls in TestItem, dict: - url = 'http://www.example.com/images/1.jpg' - item = cls({'name': 'item1', 'image_urls': [url]}) - pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'})) - requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) - results = [(True, {'url': url})] - pipeline.item_completed(results, item, None) - self.assertEqual(item['images'], [results[0][1]]) + url = 'http://www.example.com/images/1.jpg' + item = self.item_class(name='item1', image_urls=[url]) + pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'})) + requests = list(pipeline.get_media_requests(item, None)) + self.assertEqual(requests[0].url, url) + results = [(True, {'url': url})] + item = pipeline.item_completed(results, item, None) + images = ItemAdapter(item).get("images") + self.assertEqual(images, [results[0][1]]) + self.assertIsInstance(item, self.item_class) def test_item_fields_override_settings(self): - class TestItem(Item): - name = Field() - image = Field() - stored_image = Field() + url = 'http://www.example.com/images/1.jpg' + item = self.item_class(name='item1', custom_image_urls=[url]) + pipeline = ImagesPipeline.from_settings(Settings({ + 'IMAGES_STORE': 's3://example/images/', + 'IMAGES_URLS_FIELD': 'custom_image_urls', + 'IMAGES_RESULT_FIELD': 'custom_images' + })) + requests = list(pipeline.get_media_requests(item, None)) + self.assertEqual(requests[0].url, url) + results = [(True, {'url': url})] + item = pipeline.item_completed(results, item, None) + custom_images = ItemAdapter(item).get("custom_images") + self.assertEqual(custom_images, [results[0][1]]) + self.assertIsInstance(item, self.item_class) - for cls in TestItem, dict: - url = 'http://www.example.com/images/1.jpg' - item = cls({'name': 'item1', 'image': [url]}) - pipeline = ImagesPipeline.from_settings(Settings({ - 'IMAGES_STORE': 's3://example/images/', - 'IMAGES_URLS_FIELD': 'image', - 'IMAGES_RESULT_FIELD': 'stored_image' - })) - requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) - results = [(True, {'url': url})] - pipeline.item_completed(results, item, None) - self.assertEqual(item['stored_image'], [results[0][1]]) + +class ImagesPipelineTestCaseFieldsDict(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = dict + + +class ImagesPipelineTestItem(Item): + name = Field() + # default fields + image_urls = Field() + images = Field() + # overridden fields + custom_image_urls = Field() + custom_images = Field() + + +class ImagesPipelineTestCaseFieldsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = ImagesPipelineTestItem + + +@skipIf(not make_dataclass, "dataclasses module is not available") +class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if make_dataclass: + self.item_class = make_dataclass( + "FilesPipelineTestDataClass", + [ + ("name", str), + # default fields + ("image_urls", list, dataclass_field(default_factory=list)), + ("images", list, dataclass_field(default_factory=list)), + # overridden fields + ("custom_image_urls", list, dataclass_field(default_factory=list)), + ("custom_images", list, dataclass_field(default_factory=list)), + ], + ) + + +@attr.s +class ImagesPipelineTestAttrsItem: + name = attr.ib(default="") + # default fields + image_urls = attr.ib(default=lambda: []) + images = attr.ib(default=lambda: []) + # overridden fields + custom_image_urls = attr.ib(default=lambda: []) + custom_images = attr.ib(default=lambda: []) + + +class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = ImagesPipelineTestAttrsItem class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index 6dc117779..daf022aee 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -1,18 +1,25 @@ +import datetime import json import unittest -import datetime from decimal import Decimal +import attr from twisted.internet import defer -from scrapy.utils.serialize import ScrapyJSONEncoder from scrapy.http import Request, Response +from scrapy.utils.serialize import ScrapyJSONEncoder + + +try: + from dataclasses import make_dataclass +except ImportError: + make_dataclass = None class JsonEncoderTestCase(unittest.TestCase): def setUp(self): - self.encoder = ScrapyJSONEncoder() + self.encoder = ScrapyJSONEncoder(sort_keys=True) def test_encode_decode(self): dt = datetime.datetime(2010, 1, 2, 10, 11, 12) @@ -31,7 +38,8 @@ class JsonEncoderTestCase(unittest.TestCase): for input, output in [('foo', 'foo'), (d, ds), (t, ts), (dt, dts), (dec, decs), (['foo', d], ['foo', ds]), (s, ss), (dt_set, dt_sets)]: - self.assertEqual(self.encoder.encode(input), json.dumps(output)) + self.assertEqual(self.encoder.encode(input), + json.dumps(output, sort_keys=True)) def test_encode_deferred(self): self.assertIn('Deferred', self.encoder.encode(defer.Deferred())) @@ -47,3 +55,30 @@ class JsonEncoderTestCase(unittest.TestCase): rs = self.encoder.encode(r) self.assertIn(r.url, rs) self.assertIn(str(r.status), rs) + + @unittest.skipIf(not make_dataclass, "No dataclass support") + def test_encode_dataclass_item(self): + TestDataClass = make_dataclass( + "TestDataClass", + [("name", str), ("url", str), ("price", int)], + ) + item = TestDataClass(name="Product", url="http://product.org", price=1) + encoded = self.encoder.encode(item) + self.assertEqual( + encoded, + '{"name": "Product", "price": 1, "url": "http://product.org"}' + ) + + def test_encode_attrs_item(self): + @attr.s + class AttrsItem: + name = attr.ib(type=str) + url = attr.ib(type=str) + price = attr.ib(type=int) + + item = AttrsItem(name="Product", url="http://product.org", price=1) + encoded = self.encoder.encode(item) + self.assertEqual( + encoded, + '{"name": "Product", "price": 1, "url": "http://product.org"}' + ) diff --git a/tox.ini b/tox.ini index 69b1bdfdd..4c790158d 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,7 @@ deps = pytest-flake8 commands = py.test --flake8 {posargs:docs scrapy tests} - + [testenv:pylint] basepython = python3 deps = @@ -62,6 +62,7 @@ deps = -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 + itemadapter==0.1.0 lxml==3.5.0 parsel==1.5.0 Protego==0.1.15 From de4a34365a2d872eb69ab2a2edfb723658e28530 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Sun, 14 Jun 2020 22:40:49 +0530 Subject: [PATCH 104/568] fix: large data chunk not received Every data chunk received needs to be acknowledged to - update the flow control window size - get furthur data chunks from the server --- scrapy/core/http2/protocol.py | 74 +++++++++++++++++++++--------- scrapy/core/http2/stream.py | 85 ++++++++++++++++++++++++++++++----- 2 files changed, 129 insertions(+), 30 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 4036dfb3e..167596340 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,10 +1,10 @@ import logging -from typing import Dict, List from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, StreamReset, WindowUpdated + ConnectionTerminated, DataReceived, ResponseReceived, + StreamEnded, StreamReset, WindowUpdated ) from twisted.internet.protocol import connectionDone, Protocol @@ -18,33 +18,57 @@ class H2ClientProtocol(Protocol): # TODO: # 1. Check for user-agent while testing # 2. Add support for cookies - # 3. Handle priority updates + # 3. Handle priority updates (Not required) + # 4. Handle case when received events have StreamID = 0 (applied to H2Connection) + # 1 & 2: + # - Automatically handled by the Request middleware + # - request.headers will have 'Set-Cookie' value def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) + # Address of the server we are connected to + # these are updated when connection is successfully made + self.destination = None + # ID of the next request stream - # Assuming each request stream creates a new response stream - # we increment by 2 for each new request stream created + # Following the convention made by hyper-h2 each client ID + # will be odd. self.next_stream_id = 1 # Streams are stored in a dictionary keyed off their stream IDs - self.streams: Dict[int, Stream] = {} + self.streams = {} # Boolean to keep track the connection is made # If requests are received before connection is made # we keep all requests in a pool and send them as the connection # is made self.is_connection_made = False - self._pending_request_stream_pool: List[Stream] = [] + self._pending_request_stream_pool = [] + + def _stream_close_cb(self, stream_id: int): + """Called when stream is closed completely + """ + try: + del self.streams[stream_id] + except KeyError: + pass def _new_stream(self, request: Request): """Instantiates a new Stream object """ - stream = Stream(self.next_stream_id, request, self.conn) + stream_id = self.next_stream_id self.next_stream_id += 2 + stream = Stream( + stream_id=stream_id, + request=request, + connection=self.conn, + write_to_transport=self._write_to_transport, + cb_close=lambda: self._stream_close_cb(stream_id) + ) + self.streams[stream.stream_id] = stream return stream @@ -53,7 +77,6 @@ class H2ClientProtocol(Protocol): # Initiate all pending requests for stream in self._pending_request_stream_pool: stream.initiate_request() - self._write_to_transport() self._pending_request_stream_pool.clear() @@ -81,13 +104,15 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ + self.destination = self.transport.connector.getDestination() + LOGGER.info('Connection made to {}'.format(self.destination)) + self.conn.initiate_connection() self._write_to_transport() + self.is_connection_made = True self._send_pending_requests() - self.is_connection_made = True - def dataReceived(self, data): events = self.conn.receive_data(data) self._handle_events(events) @@ -95,15 +120,18 @@ class H2ClientProtocol(Protocol): def connectionLost(self, reason=connectionDone): """Called by Twisted when the transport connection is lost. + No need to write anything to transport here. """ - stream_ids = list(self.streams.keys()) + # Pop all streams which were pending and were not yet started + for stream_id in list(self.streams): + try: + self.streams[stream_id].lost_connection() + except KeyError: + pass - for stream in self._pending_request_stream_pool: - stream_ids.remove(stream.stream_id) + self.conn.close_connection() - for stream_id in stream_ids: - # TODO: Close each Stream instance in a clean manner - self.conn.end_stream(stream_id) + LOGGER.info("Connection lost with reason " + str(reason)) def _handle_events(self, events): """Private method which acts as a bridge between the events @@ -136,7 +164,7 @@ class H2ClientProtocol(Protocol): def data_received(self, event: DataReceived): stream_id = event.stream_id - self.streams[stream_id].receive_data(event.data) + self.streams[stream_id].receive_data(event.data, event.flow_controlled_length) def response_received(self, event: ResponseReceived): stream_id = event.stream_id @@ -147,9 +175,15 @@ class H2ClientProtocol(Protocol): self.streams[stream_id].end_stream() def stream_reset(self, event: StreamReset): - pass + # TODO: event.stream_id was abruptly closed + # Q. What should be the response? (Failure/Partial/???) + self.streams[event.stream_id].reset() def window_updated(self, event: WindowUpdated): stream_id = event.stream_id if stream_id != 0: - self.streams[stream_id].window_updated() + self.streams[stream_id].receive_window_update(event.delta) + else: + # TODO: + # Q. What to do when StreamID=0 ? + pass diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index b37755042..7c9cf7cf5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,3 +1,4 @@ +import logging from urllib.parse import urlparse from h2.connection import H2Connection @@ -6,6 +7,8 @@ from twisted.internet.defer import Deferred from scrapy.http import Request, Response from scrapy.http.headers import Headers +LOGGER = logging.getLogger(__name__) + class Stream: """Represents a single HTTP/2 Stream. @@ -18,17 +21,30 @@ class Stream: 1. Combine all the data frames """ - def __init__(self, stream_id: int, request: Request, connection: H2Connection): + def __init__( + self, + stream_id: int, + request: Request, + connection: H2Connection, + write_to_transport, + cb_close + ): """ Arguments: stream_id {int} -- For one HTTP/2 connection each stream is uniquely identified by a single integer request {Request} -- HTTP request connection {H2Connection} -- HTTP/2 connection this stream belongs to. + write_to_transport {callable} -- Method used to write & send data to the server + This method should be used whenever some frame is to be sent to the server. + cb_close {callable} -- Method called when this stream is closed + to notify the TCP connection instance. """ self.stream_id = stream_id self._request = request self._conn = connection + self._write_to_transport = write_to_transport + self._cb_close = cb_close self._request_body = self._request.body self.content_length = 0 if self._request_body is None else len(self._request_body) @@ -36,13 +52,20 @@ class Stream: # Each time we send a data frame, we will decrease value by the amount send. self.remaining_content_length = self.content_length - # Flag to keep track whether we have ended this stream - self.stream_ended = True + # Flag to keep track whether we have closed this stream + self.stream_closed_local = False + + # Flag to keep track whether the server has closed the stream + self.stream_closed_server = False # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. self._response_data = b"" + # The amount of data received that counts against the flow control + # window + self._response_flow_controlled_size = 0 + # Headers received after sending the request self._response_headers = Headers({}) @@ -77,6 +100,8 @@ class Stream: ] self._conn.send_headers(self.stream_id, http2_request_headers, end_stream=False) + self._write_to_transport() + self.send_data() def send_data(self): @@ -112,32 +137,59 @@ class Stream: data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size] self._conn.send_data(self.stream_id, data_chunk, end_stream=False) + self._write_to_transport() bytes_to_send = max(0, bytes_to_send - chunk_size) self.remaining_content_length = max(0, self.remaining_content_length - chunk_size) # End the stream if no more data has to be send if self.remaining_content_length == 0: + self.stream_closed_local = True self._conn.end_stream(self.stream_id) - else: - # TODO: Continue from here :) - pass - def window_updated(self): + self._write_to_transport() + + # Q. What about the rest of the data? + # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame + + def receive_window_update(self, delta): """Flow control window size was changed. Send data that earlier could not be sent as we were blocked behind the flow control. + + Arguments: + delta -- Window change delta """ - if self.remaining_content_length > 0 and not self.stream_ended: + if self.remaining_content_length > 0 and not self.stream_closed_local: self.send_data() - def receive_data(self, data: bytes): + def receive_data(self, data: bytes, flow_controlled_length: int): self._response_data += data + self._response_flow_controlled_size += flow_controlled_length + + # Acknowledge the data received + self._conn.acknowledge_received_data( + self._response_flow_controlled_size, + self.stream_id + ) def receive_headers(self, headers): for name, value in headers: self._response_headers[name] = value + def reset(self): + """Received a RST_STREAM -- forcefully reset""" + # TODO: + # Q1. Do we need to send the request again? + # Q2. What response should we send now? + self.stream_closed_server = True + self._cb_close() + + def lost_connection(self): + # TODO: Same as self.reset + self.stream_closed_server = True + self._cb_close() + def end_stream(self): """Stream is ended by the server hence no further data or headers should be expected on this stream. @@ -145,7 +197,20 @@ class Stream: We will call the response deferred callback passing the response object """ - # TODO: Set flags, certificate, ip_address + assert self.stream_closed_server is False + self.stream_closed_server = True + + self._fire_response_deferred() + self._cb_close() + + def _fire_response_deferred(self): + # TODO: + # 1. Set flags, certificate, ip_address in response + # 2. Should we fire this in case of + # 2.1 StreamReset in between when data is received partially + # 2.2 Forcefully closed the stream + + # NOTE: Presently on fired with successful response response = Response( url=self._request.url, status=self._response_headers[":status"], From 01ad8b31ab7fe86fd78a70b09a6dd61a497e0ccb Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Mon, 15 Jun 2020 05:14:00 +0530 Subject: [PATCH 105/568] refactor(http2): clean up - make separate function to parse http headers from Request instance --- scrapy/core/http2/protocol.py | 12 ++------ scrapy/core/http2/stream.py | 55 ++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 167596340..915284c33 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -50,10 +50,7 @@ class H2ClientProtocol(Protocol): def _stream_close_cb(self, stream_id: int): """Called when stream is closed completely """ - try: - del self.streams[stream_id] - except KeyError: - pass + self.streams.pop(stream_id, None) def _new_stream(self, request: Request): """Instantiates a new Stream object @@ -66,7 +63,7 @@ class H2ClientProtocol(Protocol): request=request, connection=self.conn, write_to_transport=self._write_to_transport, - cb_close=lambda: self._stream_close_cb(stream_id) + cb_close=self._stream_close_cb ) self.streams[stream.stream_id] = stream @@ -124,10 +121,7 @@ class H2ClientProtocol(Protocol): """ # Pop all streams which were pending and were not yet started for stream_id in list(self.streams): - try: - self.streams[stream_id].lost_connection() - except KeyError: - pass + self.streams[stream_id].lost_connection() self.conn.close_connection() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 7c9cf7cf5..a0b75850d 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -81,25 +81,27 @@ class Stream: """ return self._deferred_response - def initiate_request(self): - http2_request_headers = [] - for name, value in self._request.headers.items(): - http2_request_headers.append((name, value)) - + def _get_request_headers(self): url = urlparse(self._request.url) - http2_request_headers += [ - (":method", self._request.method), - (":authority", url.netloc), - # TODO: Check if scheme can be "http" for HTTP/2 ? - (":scheme", "https"), - (":path", url.path), + # Make sure pseudo-headers comes before all the other headers + headers = [ + (':method', self._request.method), + (':authority', url.netloc), - # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers - # are sent for request having body + # TODO: Check if scheme can be 'http' for HTTP/2 ? + (':scheme', 'https'), + (':path', url.path), ] - self._conn.send_headers(self.stream_id, http2_request_headers, end_stream=False) + for name, value in self._request.headers.items(): + headers.append((name, value[0])) + + return headers + + def initiate_request(self): + headers = self._get_request_headers() + self._conn.send_headers(self.stream_id, headers, end_stream=False) self._write_to_transport() self.send_data() @@ -127,23 +129,24 @@ class Stream: # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. - bytes_to_send = min(window_size, self.remaining_content_length) + bytes_to_send_size = min(window_size, self.remaining_content_length) # We now need to send a number of data frames. - while bytes_to_send > 0: - chunk_size = min(bytes_to_send, max_frame_size) + while bytes_to_send_size > 0: + chunk_size = min(bytes_to_send_size, max_frame_size) - data_chunk_start = self.content_length - self.remaining_content_length - data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size] + data_chunk_start_id = self.content_length - self.remaining_content_length + data_chunk = self._request_body[data_chunk_start_id:data_chunk_start_id + chunk_size] self._conn.send_data(self.stream_id, data_chunk, end_stream=False) - self._write_to_transport() - bytes_to_send = max(0, bytes_to_send - chunk_size) - self.remaining_content_length = max(0, self.remaining_content_length - chunk_size) + bytes_to_send_size = bytes_to_send_size - chunk_size + self.remaining_content_length = self.remaining_content_length - chunk_size # End the stream if no more data has to be send - if self.remaining_content_length == 0: + if self.remaining_content_length <= 0: + self.remaining_content_length = 0 + self.stream_closed_local = True self._conn.end_stream(self.stream_id) @@ -183,12 +186,12 @@ class Stream: # Q1. Do we need to send the request again? # Q2. What response should we send now? self.stream_closed_server = True - self._cb_close() + self._cb_close(self.stream_id) def lost_connection(self): # TODO: Same as self.reset self.stream_closed_server = True - self._cb_close() + self._cb_close(self.stream_id) def end_stream(self): """Stream is ended by the server hence no further @@ -201,7 +204,7 @@ class Stream: self.stream_closed_server = True self._fire_response_deferred() - self._cb_close() + self._cb_close(self.stream_id) def _fire_response_deferred(self): # TODO: From 214da8e5235c9676bc4b0427c8bf58f328fe6570 Mon Sep 17 00:00:00 2001 From: Ram Rachum <ram@rachum.com> Date: Wed, 17 Jun 2020 13:50:54 +0300 Subject: [PATCH 106/568] Use chain.from_iterable in python.py --- scrapy/utils/python.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 9c1f3c2fe..afa8a8135 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -334,10 +334,10 @@ class MutableChain: """ def __init__(self, *args): - self.data = chain(*args) + self.data = chain.from_iterable(args) def extend(self, *iterables): - self.data = chain(self.data, *iterables) + self.data = chain(self.data, chain.from_iterable(iterables)) def __iter__(self): return self From 3d027fb578532d504b3dbfaa77a06c3560f85d3c Mon Sep 17 00:00:00 2001 From: Stas Glubokiy <glubokiy.stas@gmail.com> Date: Wed, 17 Jun 2020 18:08:14 +0300 Subject: [PATCH 107/568] Fix missing storage.store calls in FeedExporter.close_spider (#4626) --- scrapy/extensions/feedexport.py | 4 ++- tests/test_feedexport.py | 49 ++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 998d2a5d1..30e6349d6 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -270,7 +270,9 @@ class FeedExporter: if not slot.itemcount and not slot.store_empty: # We need to call slot.storage.store nonetheless to get the file # properly closed. - return defer.maybeDeferred(slot.storage.store, slot.file) + d = defer.maybeDeferred(slot.storage.store, slot.file) + deferred_list.append(d) + continue slot.finish_exporting() logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" log_args = {'format': slot.format, diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8eeb29b6d..f7013bc44 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -7,6 +7,7 @@ import string import tempfile import warnings from io import BytesIO +from logging import getLogger from pathlib import Path from string import ascii_letters, digits from unittest import mock @@ -14,9 +15,11 @@ from urllib.parse import urljoin, urlparse, quote from urllib.request import pathname2url import lxml.etree +from testfixtures import LogCapture from twisted.internet import defer from twisted.trial import unittest -from w3lib.url import path_to_file_uri +from w3lib.url import file_uri_to_path, path_to_file_uri +from zope.interface import implementer from zope.interface.verify import verifyObject import scrapy @@ -390,6 +393,25 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): pass +@implementer(IFeedStorage) +class LogOnStoreFileStorage: + """ + This storage logs inside `store` method. + It can be used to make sure `store` method is invoked. + """ + + def __init__(self, uri): + self.path = file_uri_to_path(uri) + self.logger = getLogger() + + def open(self, spider): + return tempfile.NamedTemporaryFile(prefix='feed-') + + def store(self, file): + self.logger.info('Storage.store is called') + file.close() + + class FeedExportTest(unittest.TestCase): class MyItem(scrapy.Item): @@ -426,11 +448,17 @@ class FeedExportTest(unittest.TestCase): yield runner.crawl(spider_cls) for file_path, feed in FEEDS.items(): + if not os.path.exists(str(file_path)): + continue + with open(str(file_path), 'rb') as f: content[feed['format']] = f.read() finally: for file_path in FEEDS.keys(): + if not os.path.exists(str(file_path)): + continue + os.remove(str(file_path)) return content @@ -623,6 +651,25 @@ class FeedExportTest(unittest.TestCase): data = yield self.exported_no_data(settings) self.assertEqual(data[fmt], expctd) + @defer.inlineCallbacks + def test_export_no_items_multiple_feeds(self): + """ Make sure that `storage.store` is called for every feed. """ + settings = { + 'FEEDS': { + self._random_temp_filename(): {'format': 'json'}, + self._random_temp_filename(): {'format': 'xml'}, + self._random_temp_filename(): {'format': 'csv'}, + }, + 'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'}, + 'FEED_STORE_EMPTY': False + } + + with LogCapture() as log: + yield self.exported_no_data(settings) + + print(log) + self.assertEqual(str(log).count('Storage.store is called'), 3) + @defer.inlineCallbacks def test_export_multiple_item_classes(self): From 089dbc75e78a2da9c455f21bb3c7ebaaeb2e3582 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Wed, 17 Jun 2020 20:57:03 +0530 Subject: [PATCH 108/568] chore: use deque for pending request pool - Use itertools.count to generate next stream_id BREAKING CHANGES When sending data/body more than the local flow control window -- no window update occurs to send the remaining data frames. Hence, the complete body is not send resulting in no response received. --- scrapy/core/http2/protocol.py | 16 +++++++++------- scrapy/core/http2/stream.py | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 915284c33..dbc048ffa 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,4 +1,6 @@ +import itertools import logging +from collections import deque from h2.config import H2Configuration from h2.connection import H2Connection @@ -35,7 +37,7 @@ class H2ClientProtocol(Protocol): # ID of the next request stream # Following the convention made by hyper-h2 each client ID # will be odd. - self.next_stream_id = 1 + self.stream_id_count = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs self.streams = {} @@ -45,7 +47,7 @@ class H2ClientProtocol(Protocol): # we keep all requests in a pool and send them as the connection # is made self.is_connection_made = False - self._pending_request_stream_pool = [] + self._pending_request_stream_pool = deque() def _stream_close_cb(self, stream_id: int): """Called when stream is closed completely @@ -55,8 +57,7 @@ class H2ClientProtocol(Protocol): def _new_stream(self, request: Request): """Instantiates a new Stream object """ - stream_id = self.next_stream_id - self.next_stream_id += 2 + stream_id = next(self.stream_id_count) stream = Stream( stream_id=stream_id, @@ -72,11 +73,10 @@ class H2ClientProtocol(Protocol): def _send_pending_requests(self): # TODO: handle MAX_CONCURRENT_STREAMS # Initiate all pending requests - for stream in self._pending_request_stream_pool: + while len(self._pending_request_stream_pool): + stream = self._pending_request_stream_pool.popleft() stream.initiate_request() - self._pending_request_stream_pool.clear() - def _write_to_transport(self): """ Write data to the underlying transport connection from the HTTP2 connection instance if any @@ -108,6 +108,8 @@ class H2ClientProtocol(Protocol): self._write_to_transport() self.is_connection_made = True + # Send off all the pending requests + # as now we have established a proper HTTP/2 connection self._send_pending_requests() def dataReceived(self, data): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a0b75850d..c2e1adce5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -132,7 +132,7 @@ class Stream: bytes_to_send_size = min(window_size, self.remaining_content_length) # We now need to send a number of data frames. - while bytes_to_send_size > 0: + while bytes_to_send_size: chunk_size = min(bytes_to_send_size, max_frame_size) data_chunk_start_id = self.content_length - self.remaining_content_length @@ -163,7 +163,7 @@ class Stream: Arguments: delta -- Window change delta """ - if self.remaining_content_length > 0 and not self.stream_closed_local: + if self.stream_closed_local is False: self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int): From 700df3eeb7c540892c8b2910db688f1cc5d1c845 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Wed, 17 Jun 2020 21:02:14 +0530 Subject: [PATCH 109/568] test: mockserver with h2 protocol for tests - add Twisted[http2] in setup.py requirements - add test_protocol.py to test the current implementation BREAKING CHANGES test_download times out because of no protocol negotiated between Mockserver and HTTP/2 client --- scrapy/core/http2/test_protocol.py | 67 +++++++++++++++++++++++++++ setup.py | 6 +-- tests/test_http2_client_protocol.py | 71 +++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 scrapy/core/http2/test_protocol.py create mode 100644 tests/test_http2_client_protocol.py diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.py new file mode 100644 index 000000000..c7782a518 --- /dev/null +++ b/scrapy/core/http2/test_protocol.py @@ -0,0 +1,67 @@ +# This is simple script to test + +import json + +from twisted.internet import reactor +from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint +from twisted.internet.ssl import optionsForClientTLS + +from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.http import Request, Response, JsonRequest + +try: + with open('data.json', 'r') as f: + JSON_DATA = json.load(f) +except: + JSON_DATA = { + "data": "To test for really large amount of data -- Add data.json with lots of data.", + "why": "To test whether correct data is sent :)" + } + +# Use nghttp2 for testing whether basic setup works - for small response +HTTPBIN_AUTHORITY = u'nghttp2.org' +HTTPBIN_REQUEST_URLS = 1 * [ + Request(url='https://nghttp2.org/httpbin/get', method='GET'), + Request(url='https://nghttp2.org/httpbin/post', method='POST'), + JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA), +] + +# Use POKE_API for testing large responses +POKE_API_AUTHORITY = u'pokeapi.co' +POKE_API_REQUESTS = 15 * [ + Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'), + Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'), + Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'), + Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'), # should give 404 +] + +AUTHORITY = POKE_API_AUTHORITY +REQUEST_URLS = POKE_API_REQUESTS + +options = optionsForClientTLS( + hostname=AUTHORITY, + acceptableProtocols=[b'h2'], +) + +protocol = H2ClientProtocol() + +count_responses = 1 + + +def print_response(response): + global count_responses + assert isinstance(response, Response) + print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body))) + count_responses = count_responses + 1 + + +for request in REQUEST_URLS: + d = protocol.request(request) + d.addCallback(print_response) + +connectProtocol( + SSL4ClientEndpoint(reactor, AUTHORITY, 443, options), + protocol +) + +reactor.run() diff --git a/setup.py b/setup.py index 1b3c6771a..dafa5684a 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from os.path import dirname, join + from pkg_resources import parse_version from setuptools import setup, find_packages, __version__ as setuptools_version - with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: version = f.read().decode('ascii').strip() @@ -25,12 +25,11 @@ if has_environment_marker_platform_impl_support(): 'PyPyDispatcher>=2.1.0', ] - setup( name='Scrapy', version=version, url='https://scrapy.org', - project_urls = { + project_urls={ 'Documentation': 'https://docs.scrapy.org/', 'Source': 'https://github.com/scrapy/scrapy', 'Tracker': 'https://github.com/scrapy/scrapy/issues', @@ -69,6 +68,7 @@ setup( python_requires='>=3.5', install_requires=[ 'Twisted>=17.9.0', + 'Twisted[http2]>=17.9.0' 'cryptography>=2.0', 'cssselect>=0.9.1', 'lxml>=3.5.0', diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py new file mode 100644 index 000000000..7830f7028 --- /dev/null +++ b/tests/test_http2_client_protocol.py @@ -0,0 +1,71 @@ +import os +import shutil + +from twisted.internet import defer, reactor +from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint +from twisted.internet.ssl import optionsForClientTLS +from twisted.protocols.policies import WrappingFactory +from twisted.python.filepath import FilePath +from twisted.trial import unittest +from twisted.web import static, server + +from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.http import Request +from tests.mockserver import ssl_context_factory + + +class Http2ClientProtocolTestCase(unittest.TestCase): + scheme = 'https' + + # only used for HTTPS tests + file_key = 'keys/localhost.key' + file_certificate = 'keys/localhost.crt' + + def setUp(self): + # Start server for testing + self.path_temp = self.mktemp() + os.mkdir(self.path_temp) + FilePath(self.path_temp).child('file').setContent(b"0123456789") + r = static.File(self.path_temp) + + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.host = 'localhost' + if self.scheme is 'https': + self.port = reactor.listenSSL( + 0, self.wrapper, + ssl_context_factory(self.file_key, self.file_certificate), + interface=self.host + ) + else: + self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) + + self.port_number = self.port.getHost().port + + # Connect to the server using the custom HTTP2ClientProtocol + options = optionsForClientTLS( + hostname=self.host, + acceptableProtocols=[b'h2'] + ) + + self.protocol = H2ClientProtocol() + + connectProtocol( + endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options), + protocol=self.protocol + ) + + def getURL(self, path): + return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path) + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + shutil.rmtree(self.path_temp) + + def test_download(self): + request = Request(self.getURL('file')) + d = self.protocol.request(request) + d.addCallback(lambda response: response.body) + d.addCallback(self.assertEqual, b"0123456789") + return d From 5d541731870eaaa3dd658673b82a7d0ca056f689 Mon Sep 17 00:00:00 2001 From: Devi Sandeep <sandeep0138@gmail.com> Date: Thu, 18 Jun 2020 05:01:38 -0500 Subject: [PATCH 110/568] Update docs on accessing callback arguments in errback (#4634) --- docs/topics/request-response.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index bbd715766..d88d40b00 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -189,6 +189,10 @@ Request objects cloned using the ``copy()`` or ``replace()`` methods, and can also be accessed, in your spider, from the ``response.cb_kwargs`` attribute. + In case of a failure to process the request, this dict can be accessed as + ``failure.request.cb_kwargs`` in the request's errback. For more information, + see :ref:`topics-request-response-ref-accessing-callback-arguments-in-errback`. + .. method:: Request.copy() Return a new Request which is a copy of this Request. See also: @@ -312,6 +316,31 @@ errors if needed:: request = failure.request self.logger.error('TimeoutError on %s', request.url) +.. _topics-request-response-ref-accessing-callback-arguments-in-errback: + +Accessing additional data in errback functions +---------------------------------------------- + +In case of a failure to process the request, you may be interested in +accessing arguments to the callback functions so you can process further +based on the arguments in the errback. The following example shows how to +achieve this by using ``Failure.request.cb_kwargs``:: + + def parse(self, response): + request = scrapy.Request('http://www.example.com/index.html', + callback=self.parse_page2, + errback=self.errback_page2, + cb_kwargs=dict(main_url=response.url)) + yield request + + def parse_page2(self, response, main_url): + pass + + def errback_page2(self, failure): + yield dict( + main_url=failure.request.cb_kwargs['main_url'], + ) + .. _topics-request-meta: Request.meta special keys From 7babf359e0221613b872b5b204e7c523b7b84486 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Thu, 18 Jun 2020 13:52:04 -0300 Subject: [PATCH 111/568] Typing: Tox env, CI job --- .gitignore | 1 + .travis.yml | 2 + setup.cfg | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++++ tox.ini | 7 +++ 4 files changed, 181 insertions(+) diff --git a/.gitignore b/.gitignore index ff6e2ea65..83a2569dd 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ htmlcov/ .pytest_cache/ .coverage.* .cache/ +.mypy_cache/ # Windows Thumbs.db diff --git a/.travis.yml b/.travis.yml index e44f85237..b403ac54c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,8 @@ matrix: python: 3.8 - env: TOXENV=docs python: 3.7 # Keep in sync with .readthedocs.yml + - env: TOXENV=typing + python: 3.8 - env: TOXENV=pypy3 - env: TOXENV=pinned diff --git a/setup.cfg b/setup.cfg index 2296a1052..a9138c1c0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,3 +3,174 @@ doc_files = docs AUTHORS INSTALL LICENSE README.rst [bdist_wheel] universal=1 + +[mypy] +ignore_missing_imports = true +follow_imports = skip + +# FIXME: remove the following sections once the issues are solved + +[mypy-scrapy] +ignore_errors = True + +[mypy-scrapy._monkeypatches] +ignore_errors = True + +[mypy-scrapy.commands] +ignore_errors = True + +[mypy-scrapy.commands.bench] +ignore_errors = True + +[mypy-scrapy.commands.parse] +ignore_errors = True + +[mypy-scrapy.downloadermiddlewares.httpproxy] +ignore_errors = True + +[mypy-scrapy.contracts] +ignore_errors = True + +[mypy-scrapy.core.spidermw] +ignore_errors = True + +[mypy-scrapy.interfaces] +ignore_errors = True + +[mypy-scrapy.item] +ignore_errors = True + +[mypy-scrapy.http.cookies] +ignore_errors = True + +[mypy-scrapy.mail] +ignore_errors = True + +[mypy-scrapy.pipelines.images] +ignore_errors = True + +[mypy-scrapy.settings.default_settings] +ignore_errors = True + +[mypy-scrapy.spidermiddlewares.referer] +ignore_errors = True + +[mypy-scrapy.utils.httpobj] +ignore_errors = True + +[mypy-scrapy.utils.request] +ignore_errors = True + +[mypy-scrapy.utils.response] +ignore_errors = True + +[mypy-scrapy.utils.spider] +ignore_errors = True + +[mypy-scrapy.utils.trackref] +ignore_errors = True + +[mypy-tests.mocks.dummydbm] +ignore_errors = True + +[mypy-tests.spiders] +ignore_errors = True + +[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception] +ignore_errors = True + +[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal] +ignore_errors = True + +[mypy-tests.test_command_fetch] +ignore_errors = True + +[mypy-tests.test_command_parse] +ignore_errors = True + +[mypy-tests.test_command_shell] +ignore_errors = True + +[mypy-tests.test_command_version] +ignore_errors = True + +[mypy-tests.test_contracts] +ignore_errors = True + +[mypy-tests.test_crawler] +ignore_errors = True + +[mypy-tests.test_downloader_handlers] +ignore_errors = True + +[mypy-tests.test_engine] +ignore_errors = True + +[mypy-tests.test_exporters] +ignore_errors = True + +[mypy-tests.test_http_request] +ignore_errors = True + +[mypy-tests.test_linkextractors] +ignore_errors = True + +[mypy-tests.test_loader] +ignore_errors = True + +[mypy-tests.test_pipeline_crawl] +ignore_errors = True + +[mypy-tests.test_pipeline_files] +ignore_errors = True + +[mypy-tests.test_pipeline_images] +ignore_errors = True + +[mypy-tests.test_pipelines] +ignore_errors = True + +[mypy-tests.test_request_cb_kwargs] +ignore_errors = True + +[mypy-tests.test_request_left] +ignore_errors = True + +[mypy-tests.test_scheduler] +ignore_errors = True + +[mypy-tests.test_signals] +ignore_errors = True + +[mypy-tests.test_spiderloader.test_spiders.nested.spider4] +ignore_errors = True + +[mypy-tests.test_spiderloader.test_spiders.spider1] +ignore_errors = True + +[mypy-tests.test_spiderloader.test_spiders.spider2] +ignore_errors = True + +[mypy-tests.test_spiderloader.test_spiders.spider3] +ignore_errors = True + +[mypy-tests.test_spidermiddleware_httperror] +ignore_errors = True + +[mypy-tests.test_spidermiddleware_output_chain] +ignore_errors = True + +[mypy-tests.test_spidermiddleware_referer] +ignore_errors = True + +[mypy-tests.test_utils_reqser] +ignore_errors = True + +[mypy-tests.test_utils_serialize] +ignore_errors = True + +[mypy-tests.test_utils_spider] +ignore_errors = True + +[mypy-tests.test_utils_url] +ignore_errors = True diff --git a/tox.ini b/tox.ini index 4c790158d..27d21ade2 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,13 @@ passenv = commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} +[testenv:typing] +basepython = python3 +deps = + mypy==0.780 +commands = + mypy {posargs: scrapy tests} + [testenv:security] basepython = python3 deps = From a4bfd5ab6fd75c4badac1c5d9b40706181c41bd9 Mon Sep 17 00:00:00 2001 From: Stanislau Hluboki <glubokiy.stas@gmail.com> Date: Sat, 13 Jun 2020 18:04:38 +0300 Subject: [PATCH 112/568] Fix duplicated feed logs --- scrapy/extensions/feedexport.py | 19 +++++++--- tests/test_feedexport.py | 63 +++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 30e6349d6..61dad8726 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -279,11 +279,20 @@ class FeedExporter: 'itemcount': slot.itemcount, 'uri': slot.uri} d = defer.maybeDeferred(slot.storage.store, slot.file) - d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args, - extra={'spider': spider})) - d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args, - exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + + # Use `largs=log_args` to copy log_args into function's scope + # instead of using `log_args` from the outer scope + d.addCallback( + lambda _, largs=log_args: logger.info( + logfmt % "Stored", largs, extra={'spider': spider} + ) + ) + d.addErrback( + lambda f, largs=log_args: logger.error( + logfmt % "Error storing", largs, + exc_info=failure_to_exc_info(f), extra={'spider': spider} + ) + ) deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f7013bc44..e38644214 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -393,6 +393,27 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): pass +class DummyBlockingFeedStorage(BlockingFeedStorage): + + def __init__(self, uri): + self.path = file_uri_to_path(uri) + + def _store_in_thread(self, file): + dirname = os.path.dirname(self.path) + if dirname and not os.path.exists(dirname): + os.makedirs(dirname) + with open(self.path, 'ab') as output_file: + output_file.write(file.read()) + + file.close() + + +class FailingBlockingFeedStorage(DummyBlockingFeedStorage): + + def _store_in_thread(self, file): + raise OSError('Cannot store') + + @implementer(IFeedStorage) class LogOnStoreFileStorage: """ @@ -1025,3 +1046,45 @@ class FeedExportTest(unittest.TestCase): } data = yield self.exported_no_data(settings) self.assertEqual(data['csv'], b'') + + @defer.inlineCallbacks + def test_multiple_feeds_success_logs_blocking_feed_storage(self): + settings = { + 'FEEDS': { + self._random_temp_filename(): {'format': 'json'}, + self._random_temp_filename(): {'format': 'xml'}, + self._random_temp_filename(): {'format': 'csv'}, + }, + 'FEED_STORAGES': {'file': 'tests.test_feedexport.DummyBlockingFeedStorage'}, + } + items = [ + {'foo': 'bar1', 'baz': ''}, + {'foo': 'bar2', 'baz': 'quux'}, + ] + with LogCapture() as log: + yield self.exported_data(items, settings) + + print(log) + for fmt in ['json', 'xml', 'csv']: + self.assertIn('Stored %s feed (2 items)' % fmt, str(log)) + + @defer.inlineCallbacks + def test_multiple_feeds_failing_logs_blocking_feed_storage(self): + settings = { + 'FEEDS': { + self._random_temp_filename(): {'format': 'json'}, + self._random_temp_filename(): {'format': 'xml'}, + self._random_temp_filename(): {'format': 'csv'}, + }, + 'FEED_STORAGES': {'file': 'tests.test_feedexport.FailingBlockingFeedStorage'}, + } + items = [ + {'foo': 'bar1', 'baz': ''}, + {'foo': 'bar2', 'baz': 'quux'}, + ] + with LogCapture() as log: + yield self.exported_data(items, settings) + + print(log) + for fmt in ['json', 'xml', 'csv']: + self.assertIn('Error storing %s feed (2 items)' % fmt, str(log)) From b99fe4aa4c35ff8f5c2355a09988ddb1e90d6b70 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Fri, 19 Jun 2020 21:41:15 -0300 Subject: [PATCH 113/568] Add google-cloud-storage to the 'pinned' tox environment --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 4c790158d..d28eab131 100644 --- a/tox.ini +++ b/tox.ini @@ -76,6 +76,7 @@ deps = -rtests/requirements-py3.txt # Extras botocore==1.3.23 + google-cloud-storage==1.29.0 Pillow==3.4.2 [testenv:extra-deps] From 303485a9b4fd86c5123c96c81a9401b5e323a91d Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Sun, 21 Jun 2020 00:33:34 +0530 Subject: [PATCH 114/568] fix(http2): POST request not sending large body --- scrapy/core/http2/protocol.py | 51 +++++++----- scrapy/core/http2/stream.py | 148 ++++++++++++++++++++++++---------- 2 files changed, 138 insertions(+), 61 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index dbc048ffa..d6134183a 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,3 +1,4 @@ +import ipaddress import itertools import logging from collections import deque @@ -5,7 +6,7 @@ from collections import deque from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - ConnectionTerminated, DataReceived, ResponseReceived, + DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) from twisted.internet.protocol import connectionDone, Protocol @@ -49,6 +50,13 @@ class H2ClientProtocol(Protocol): self.is_connection_made = False self._pending_request_stream_pool = deque() + # Some meta data of this connection + # initialized when connection is successfully made + self._metadata = { + 'certificate': None, + 'ip_address': None + } + def _stream_close_cb(self, stream_id: int): """Called when stream is closed completely """ @@ -63,6 +71,7 @@ class H2ClientProtocol(Protocol): stream_id=stream_id, request=request, connection=self.conn, + metadata=self._metadata, write_to_transport=self._write_to_transport, cb_close=self._stream_close_cb ) @@ -73,7 +82,7 @@ class H2ClientProtocol(Protocol): def _send_pending_requests(self): # TODO: handle MAX_CONCURRENT_STREAMS # Initiate all pending requests - while len(self._pending_request_stream_pool): + while self._pending_request_stream_pool: stream = self._pending_request_stream_pool.popleft() stream.initiate_request() @@ -84,6 +93,8 @@ class H2ClientProtocol(Protocol): data = self.conn.data_to_send() self.transport.write(data) + LOGGER.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address'])) + def request(self, _request: Request): stream = self._new_stream(_request) d = stream.get_response() @@ -101,17 +112,16 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ - self.destination = self.transport.connector.getDestination() + self.destination = self.transport.getPeer() LOGGER.info('Connection made to {}'.format(self.destination)) + self._metadata['certificate'] = self.transport.getPeerCertificate() + self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) + self.conn.initiate_connection() self._write_to_transport() self.is_connection_made = True - # Send off all the pending requests - # as now we have established a proper HTTP/2 connection - self._send_pending_requests() - def dataReceived(self, data): events = self.conn.receive_data(data) self._handle_events(events) @@ -123,7 +133,7 @@ class H2ClientProtocol(Protocol): """ # Pop all streams which were pending and were not yet started for stream_id in list(self.streams): - self.streams[stream_id].lost_connection() + self.streams[stream_id].close() self.conn.close_connection() @@ -139,9 +149,7 @@ class H2ClientProtocol(Protocol): """ for event in events: LOGGER.debug(event) - if isinstance(event, ConnectionTerminated): - self.connection_terminated(event) - elif isinstance(event, DataReceived): + if isinstance(event, DataReceived): self.data_received(event) elif isinstance(event, ResponseReceived): self.response_received(event) @@ -151,13 +159,12 @@ class H2ClientProtocol(Protocol): self.stream_reset(event) elif isinstance(event, WindowUpdated): self.window_updated(event) + elif isinstance(event, SettingsAcknowledged): + self.settings_acknowledged(event) else: LOGGER.info("Received unhandled event {}".format(event)) # Event handler functions starts here - def connection_terminated(self, event: ConnectionTerminated): - pass - def data_received(self, event: DataReceived): stream_id = event.stream_id self.streams[stream_id].receive_data(event.data, event.flow_controlled_length) @@ -166,20 +173,26 @@ class H2ClientProtocol(Protocol): stream_id = event.stream_id self.streams[stream_id].receive_headers(event.headers) + def settings_acknowledged(self, event: SettingsAcknowledged): + # Send off all the pending requests + # as now we have established a proper HTTP/2 connection + self._send_pending_requests() + def stream_ended(self, event: StreamEnded): stream_id = event.stream_id - self.streams[stream_id].end_stream() + self.streams[stream_id].close() def stream_reset(self, event: StreamReset): # TODO: event.stream_id was abruptly closed # Q. What should be the response? (Failure/Partial/???) - self.streams[event.stream_id].reset() + self.streams[event.stream_id].close(event) def window_updated(self, event: WindowUpdated): stream_id = event.stream_id if stream_id != 0: self.streams[stream_id].receive_window_update(event.delta) else: - # TODO: - # Q. What to do when StreamID=0 ? - pass + # Send leftover data for all the streams + for stream in self.streams.values(): + if stream.request_sent: + stream.send_data() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index c2e1adce5..e8b4471d6 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,11 +1,15 @@ import logging +from typing import Dict from urllib.parse import urlparse from h2.connection import H2Connection +from h2.events import StreamEnded +from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred -from scrapy.http import Request, Response +from scrapy.http import Request from scrapy.http.headers import Headers +from scrapy.responsetypes import responsetypes LOGGER = logging.getLogger(__name__) @@ -22,12 +26,13 @@ class Stream: """ def __init__( - self, - stream_id: int, - request: Request, - connection: H2Connection, - write_to_transport, - cb_close + self, + stream_id: int, + request: Request, + connection: H2Connection, + metadata: Dict, + write_to_transport, + cb_close ): """ Arguments: @@ -35,6 +40,7 @@ class Stream: uniquely identified by a single integer request {Request} -- HTTP request connection {H2Connection} -- HTTP/2 connection this stream belongs to. + metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection write_to_transport {callable} -- Method used to write & send data to the server This method should be used whenever some frame is to be sent to the server. cb_close {callable} -- Method called when this stream is closed @@ -43,12 +49,16 @@ class Stream: self.stream_id = stream_id self._request = request self._conn = connection + self._metadata = metadata self._write_to_transport = write_to_transport self._cb_close = cb_close self._request_body = self._request.body self.content_length = 0 if self._request_body is None else len(self._request_body) + # Flag to keep track whether this stream has initiated the request + self.request_sent = False + # Each time we send a data frame, we will decrease value by the amount send. self.remaining_content_length = self.content_length @@ -58,20 +68,30 @@ class Stream: # Flag to keep track whether the server has closed the stream self.stream_closed_server = False - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. - self._response_data = b"" - # The amount of data received that counts against the flow control # window self._response_flow_controlled_size = 0 - # Headers received after sending the request - self._response_headers = Headers({}) + # Private variable used to build the response + # this response is then converted to appropriate Response class + # passed to the response deferred callback + self._response = { + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. + 'body': b'', + + # Headers received after sending the request + 'headers': Headers({}) + } # TODO: Add canceller for the Deferred below self._deferred_response = Deferred() + def __str__(self): + return "Stream(id={})".format(self.stream_id) + + __repr__ = __str__ + def get_response(self): """Simply return a Deferred which fires when response from the asynchronous request is available @@ -104,6 +124,8 @@ class Stream: self._conn.send_headers(self.stream_id, headers, end_stream=False) self._write_to_transport() + self.request_sent = True + self.send_data() def send_data(self): @@ -112,7 +134,18 @@ class Stream: If the content length is 0 initially then we end the stream immediately and wait for response data. + + Warning: Only call this method when stream not closed from client side + and has initiated request already by sending HEADER frame. If not then + stream will be closed from client side with 499 response. + + TODO: Q. Should we instead raise ProtocolError here with a proper message? """ + if self.stream_closed_local or self.stream_closed_server: + raise StreamClosedError(self.stream_id) + elif not self.request_sent: + self.close() + return # TODO: # 1. Add test for sending very large data @@ -132,7 +165,8 @@ class Stream: bytes_to_send_size = min(window_size, self.remaining_content_length) # We now need to send a number of data frames. - while bytes_to_send_size: + data_frames_sent = 0 + while bytes_to_send_size > 0: chunk_size = min(bytes_to_send_size, max_frame_size) data_chunk_start_id = self.content_length - self.remaining_content_length @@ -140,16 +174,24 @@ class Stream: self._conn.send_data(self.stream_id, data_chunk, end_stream=False) + data_frames_sent += 1 bytes_to_send_size = bytes_to_send_size - chunk_size self.remaining_content_length = self.remaining_content_length - chunk_size - # End the stream if no more data has to be send - if self.remaining_content_length <= 0: - self.remaining_content_length = 0 + self.remaining_content_length = max(0, self.remaining_content_length) + LOGGER.debug("{} sending {}/{} data bytes ({} frames) to {}".format( + self, + self.content_length - self.remaining_content_length, self.content_length, + data_frames_sent, + self._metadata['ip_address']) + ) + # End the stream if no more data needs to be send + if self.remaining_content_length == 0: self.stream_closed_local = True self._conn.end_stream(self.stream_id) + # Write data to transport -- Empty the outstanding data self._write_to_transport() # Q. What about the rest of the data? @@ -163,11 +205,11 @@ class Stream: Arguments: delta -- Window change delta """ - if self.stream_closed_local is False: + if self.remaining_content_length > 0 and not self.stream_closed_server: self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int): - self._response_data += data + self._response['body'] += data self._response_flow_controlled_size += flow_controlled_length # Acknowledge the data received @@ -178,47 +220,69 @@ class Stream: def receive_headers(self, headers): for name, value in headers: - self._response_headers[name] = value + self._response['headers'][name] = value - def reset(self): - """Received a RST_STREAM -- forcefully reset""" - # TODO: - # Q1. Do we need to send the request again? - # Q2. What response should we send now? - self.stream_closed_server = True - self._cb_close(self.stream_id) + def close(self, event=None): + """Based on the event sent we will handle each case. - def lost_connection(self): - # TODO: Same as self.reset - self.stream_closed_server = True - self._cb_close(self.stream_id) - - def end_stream(self): - """Stream is ended by the server hence no further + event: StreamEnded + Stream is ended by the server hence no further data or headers should be expected on this stream. - We will call the response deferred callback passing the response object + + event: StreamReset + Stream reset via RST_FRAME by the upstream hence forcefully close + this stream and send TODO: ? + + event: None + No event is launched -- Hence we will simply close this stream """ + # TODO: In case of abruptly stream close + # Q1. Do we need to send the request again? + # Q2. What response should we send now? assert self.stream_closed_server is False self.stream_closed_server = True + if not isinstance(event, StreamEnded): + # TODO + # Stream was abruptly ended here + # Partial - Content-Length header not provided + pass + self._fire_response_deferred() self._cb_close(self.stream_id) - def _fire_response_deferred(self): + def _fire_response_deferred(self, flags=None): + """Builds response from the self._response dict + and fires the response deferred callback with the + generated response instance""" # TODO: # 1. Set flags, certificate, ip_address in response # 2. Should we fire this in case of # 2.1 StreamReset in between when data is received partially # 2.2 Forcefully closed the stream + # 3. Update Client Side Status Codes here - # NOTE: Presently on fired with successful response - response = Response( + response_cls = responsetypes.from_args( + headers=self._response['headers'], url=self._request.url, - status=self._response_headers[":status"], - headers=self._response_headers, - body=self._response_data, - request=self._request + body=self._response['body'] ) + + # If there is :status in headers then + # HTTP Status Code: 499 - Client Closed Request + status = self._response['headers'].get(':status', '499') + + response = response_cls( + url=self._request.url, + status=status, + headers=self._response['headers'], + body=self._response['body'], + request=self._request, + flags=flags, + certificate=self._metadata['certificate'], + ip_address=self._metadata['ip_address'] + ) + self._deferred_response.callback(response) From c74ef660c7b28cd69825a4b9843230a383f702f4 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Sun, 21 Jun 2020 09:34:23 +0530 Subject: [PATCH 115/568] feat: handle response for different reasons - Add StreamCloseReason enum - Send response for different cases considering download_warnsize, download_maxsize, fail_on_data_loss, connection lost, etc. --- scrapy/core/http2/protocol.py | 15 ++- scrapy/core/http2/stream.py | 209 ++++++++++++++++++++++++---------- 2 files changed, 155 insertions(+), 69 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index d6134183a..188c14c15 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -11,7 +11,7 @@ from h2.events import ( ) from twisted.internet.protocol import connectionDone, Protocol -from scrapy.core.http2.stream import Stream +from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.http import Request LOGGER = logging.getLogger(__name__) @@ -71,7 +71,7 @@ class H2ClientProtocol(Protocol): stream_id=stream_id, request=request, connection=self.conn, - metadata=self._metadata, + conn_metadata=self._metadata, write_to_transport=self._write_to_transport, cb_close=self._stream_close_cb ) @@ -133,7 +133,7 @@ class H2ClientProtocol(Protocol): """ # Pop all streams which were pending and were not yet started for stream_id in list(self.streams): - self.streams[stream_id].close() + self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST) self.conn.close_connection() @@ -180,19 +180,18 @@ class H2ClientProtocol(Protocol): def stream_ended(self, event: StreamEnded): stream_id = event.stream_id - self.streams[stream_id].close() + self.streams[stream_id].close(StreamCloseReason.ENDED) def stream_reset(self, event: StreamReset): # TODO: event.stream_id was abruptly closed # Q. What should be the response? (Failure/Partial/???) - self.streams[event.stream_id].close(event) + self.streams[event.stream_id].close(StreamCloseReason.RESET) def window_updated(self, event: WindowUpdated): stream_id = event.stream_id if stream_id != 0: - self.streams[stream_id].receive_window_update(event.delta) + self.streams[stream_id].receive_window_update() else: # Send leftover data for all the streams for stream in self.streams.values(): - if stream.request_sent: - stream.send_data() + stream.receive_window_update() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index e8b4471d6..07a4428c8 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,11 +1,15 @@ import logging +from enum import IntFlag, auto +from io import BytesIO from typing import Dict from urllib.parse import urlparse from h2.connection import H2Connection -from h2.events import StreamEnded +from h2.errors import ErrorCodes from h2.exceptions import StreamClosedError -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, CancelledError +from twisted.python.failure import Failure +from twisted.web.client import ResponseFailed from scrapy.http import Request from scrapy.http.headers import Headers @@ -14,6 +18,20 @@ from scrapy.responsetypes import responsetypes LOGGER = logging.getLogger(__name__) +class StreamCloseReason(IntFlag): + # Received a StreamEnded event + ENDED = auto() + + # Received a StreamReset event -- ended abruptly + RESET = auto() + + # Transport connection was lost + CONNECTION_LOST = auto() + + # Expected response body size is more than allowed limit + MAXSIZE_EXCEEDED = auto() + + class Stream: """Represents a single HTTP/2 Stream. @@ -26,13 +44,16 @@ class Stream: """ def __init__( - self, - stream_id: int, - request: Request, - connection: H2Connection, - metadata: Dict, - write_to_transport, - cb_close + self, + stream_id: int, + request: Request, + connection: H2Connection, + conn_metadata: Dict, + write_to_transport, + cb_close, + download_maxsize=0, + download_warnsize=0, + fail_on_data_loss=True ): """ Arguments: @@ -40,7 +61,7 @@ class Stream: uniquely identified by a single integer request {Request} -- HTTP request connection {H2Connection} -- HTTP/2 connection this stream belongs to. - metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection + conn_metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection write_to_transport {callable} -- Method used to write & send data to the server This method should be used whenever some frame is to be sent to the server. cb_close {callable} -- Method called when this stream is closed @@ -49,16 +70,24 @@ class Stream: self.stream_id = stream_id self._request = request self._conn = connection - self._metadata = metadata + self._conn_metadata = conn_metadata self._write_to_transport = write_to_transport self._cb_close = cb_close - self._request_body = self._request.body - self.content_length = 0 if self._request_body is None else len(self._request_body) + self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) + self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) + self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss) + + self.request_start_time = None + + self.content_length = 0 if self._request.body is None else len(self._request.body) # Flag to keep track whether this stream has initiated the request self.request_sent = False + # Flag to track whether we have logged about exceeding download warnsize + self._reached_warnsize = False + # Each time we send a data frame, we will decrease value by the amount send. self.remaining_content_length = self.content_length @@ -68,17 +97,17 @@ class Stream: # Flag to keep track whether the server has closed the stream self.stream_closed_server = False - # The amount of data received that counts against the flow control - # window - self._response_flow_controlled_size = 0 - # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback self._response = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. - 'body': b'', + 'body': BytesIO(), + + # The amount of data received that counts against the flow control + # window + 'flow_controlled_size': 0, # Headers received after sending the request 'headers': Headers({}) @@ -137,16 +166,8 @@ class Stream: Warning: Only call this method when stream not closed from client side and has initiated request already by sending HEADER frame. If not then - stream will be closed from client side with 499 response. - - TODO: Q. Should we instead raise ProtocolError here with a proper message? + stream will raise ProtocolError (raise by h2 state machine). """ - if self.stream_closed_local or self.stream_closed_server: - raise StreamClosedError(self.stream_id) - elif not self.request_sent: - self.close() - return - # TODO: # 1. Add test for sending very large data # 2. Add test for small data @@ -154,6 +175,9 @@ class Stream: # 3.1 Large number of request # 3.2 Small number of requests + if self.stream_closed_local: + raise StreamClosedError(self.stream_id) + # Firstly, check what the flow control window is for current stream. window_size = self._conn.local_flow_control_window(stream_id=self.stream_id) @@ -170,7 +194,7 @@ class Stream: chunk_size = min(bytes_to_send_size, max_frame_size) data_chunk_start_id = self.content_length - self.remaining_content_length - data_chunk = self._request_body[data_chunk_start_id:data_chunk_start_id + chunk_size] + data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size] self._conn.send_data(self.stream_id, data_chunk, end_stream=False) @@ -183,7 +207,7 @@ class Stream: self, self.content_length - self.remaining_content_length, self.content_length, data_frames_sent, - self._metadata['ip_address']) + self._conn_metadata['ip_address']) ) # End the stream if no more data needs to be send @@ -197,24 +221,40 @@ class Stream: # Q. What about the rest of the data? # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame - def receive_window_update(self, delta): + def receive_window_update(self): """Flow control window size was changed. Send data that earlier could not be sent as we were blocked behind the flow control. - - Arguments: - delta -- Window change delta """ - if self.remaining_content_length > 0 and not self.stream_closed_server: + if self.remaining_content_length and not self.stream_closed_server and self.request_sent: self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int): - self._response['body'] += data - self._response_flow_controlled_size += flow_controlled_length + self._response['body'].write(data) + self._response['flow_controlled_size'] += flow_controlled_length + + if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize: + # Clear buffer earlier to avoid keeping data in memory for a long time + self._response['body'].truncate(0) + self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) + return + + if self._download_warnsize \ + and self._response['flow_controlled_size'] > self._download_warnsize \ + and not self._reached_warnsize: + self._reached_warnsize = True + warning_msg = ('Received more ({bytes}) bytes than download ', + 'warn size ({warnsize}) in request {request}') + warning_args = { + 'bytes': self._response['flow_controlled_size'], + 'warnsize': self._download_warnsize, + 'request': self._request + } + LOGGER.warning(warning_msg, warning_args) # Acknowledge the data received self._conn.acknowledge_received_data( - self._response_flow_controlled_size, + self._response['flow_controlled_size'], self.stream_id ) @@ -222,35 +262,83 @@ class Stream: for name, value in headers: self._response['headers'][name] = value - def close(self, event=None): - """Based on the event sent we will handle each case. + # Check if we exceed the allowed max data size which can be received + expected_size = int(self._response['headers'].get(b'Content-Length', -1)) + if self._download_maxsize and expected_size > self._download_maxsize: + self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) + return - event: StreamEnded - Stream is ended by the server hence no further - data or headers should be expected on this stream. - We will call the response deferred callback passing - the response object + if self._download_warnsize and expected_size > self._download_warnsize: + warning_msg = ("Expected response size ({size}) larger than ", + "download warn size ({warnsize}) in request {request}.") + warning_args = { + 'size': expected_size, 'warnsize': self._download_warnsize, + 'request': self._request + } + LOGGER.warning(warning_msg, warning_args) - event: StreamReset - Stream reset via RST_FRAME by the upstream hence forcefully close - this stream and send TODO: ? + def reset_stream(self, reason=StreamCloseReason.RESET): + """Close this stream by sending a RST_FRAME to the remote peer""" + # TODO: Q. REFUSED_STREAM or CANCEL ? + if self.stream_closed_local: + raise StreamClosedError(self.stream_id) - event: None - No event is launched -- Hence we will simply close this stream + self.stream_closed_local = True + self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) + self._write_to_transport() + self.close(reason) + + def _is_data_lost(self) -> bool: + assert self.stream_closed_server + + expected_size = self._response['flow_controlled_size'] + received_body_size = int(self._response['headers'][b'Content-Length']) + + return expected_size != received_body_size + + def close(self, reason: StreamCloseReason): + """Based on the reason sent we will handle each case. """ # TODO: In case of abruptly stream close # Q1. Do we need to send the request again? # Q2. What response should we send now? - assert self.stream_closed_server is False + if self.stream_closed_server: + raise StreamClosedError(self.stream_id) + self.stream_closed_server = True - if not isinstance(event, StreamEnded): - # TODO - # Stream was abruptly ended here - # Partial - Content-Length header not provided - pass + flags = None + if b'Content-Length' not in self._response['headers']: + # Missing Content-Length - PotentialDataLoss + flags = ['partial'] + elif self._is_data_lost(): + if self._fail_on_dataloss: + self._deferred_response.errback(ResponseFailed([Failure()])) + self._cb_close(self.stream_id) + return + else: + flags = ['dataloss'] + + if reason is StreamCloseReason.ENDED: + self._fire_response_deferred(flags) + + elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST): + # Stream was abruptly ended here + self._deferred_response.errback(ResponseFailed([Failure()])) + + elif reason is StreamCloseReason.MAXSIZE_EXCEEDED: + expected_size = int(self._response['headers'].get(b'Content-Length', -1)) + error_msg = ("Cancelling download of {url}: expected response " + "size ({size}) larger than download max size ({maxsize}).") + error_args = { + 'url': self._request.url, + 'size': expected_size, + 'maxsize': self._download_maxsize + } + + LOGGER.error(error_msg, error_args) + self._deferred_response.errback(CancelledError(error_msg.format(**error_args))) - self._fire_response_deferred() self._cb_close(self.stream_id) def _fire_response_deferred(self, flags=None): @@ -258,7 +346,6 @@ class Stream: and fires the response deferred callback with the generated response instance""" # TODO: - # 1. Set flags, certificate, ip_address in response # 2. Should we fire this in case of # 2.1 StreamReset in between when data is received partially # 2.2 Forcefully closed the stream @@ -270,7 +357,7 @@ class Stream: body=self._response['body'] ) - # If there is :status in headers then + # If there is no :status in headers then # HTTP Status Code: 499 - Client Closed Request status = self._response['headers'].get(':status', '499') @@ -278,11 +365,11 @@ class Stream: url=self._request.url, status=status, headers=self._response['headers'], - body=self._response['body'], + body=self._response['body'].getvalue(), request=self._request, flags=flags, - certificate=self._metadata['certificate'], - ip_address=self._metadata['ip_address'] + certificate=self._conn_metadata['certificate'], + ip_address=self._conn_metadata['ip_address'] ) self._deferred_response.callback(response) From 3efea98e0518cd0d3b92b06f6b44b701dff1e53d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Mon, 22 Jun 2020 12:41:14 -0300 Subject: [PATCH 116/568] Docs: add note about dataclass items and loaders --- docs/topics/loaders.rst | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 6645bf123..d70e03ad4 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -26,7 +26,7 @@ Using Item Loaders to populate items To use an Item Loader, you must first instantiate it. You can either instantiate it with an :ref:`item object <topics-items>` or without one, in which case an instance of :class:`~scrapy.item.Item` is automatically created in the -Item Loader ``__init__`` method using the :class:`~scrapy.item.Item` subclass +Item Loader ``__init__`` method using the :ref:`item <topics-items>` class specified in the :attr:`ItemLoader.default_item_class` attribute. Then, you start collecting values into the Item Loader, typically using @@ -76,6 +76,41 @@ called which actually returns the item populated with the data previously extracted and collected with the :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`, and :meth:`~ItemLoader.add_value` calls. + +.. _topics-loaders-dataclass: + +Working with dataclass items +============================ + +By default, :ref:`dataclass items <dataclass-items>` require all fields to be +passed when created. This could be an issue when using dataclass items with +item loaders, since fields could be populated incrementally. + +Given the way that item loaders store data internally, the recommended approach +to overcome this is to define items using the :func:`~dataclasses.field` +function, with ``list`` as the ``default_factory`` argument:: + + from dataclasses import dataclass, field + + @dataclass + class InventoryItem: + name: str = field(default_factory=list) + price: float = field(default_factory=list) + stock: int = field(default_factory=list) + +Note that in order to keep the example simple, the types do not match +completely. A more accurate but verbose definition would be:: + + from dataclasses import dataclass, field + from typing import List, Union + + @dataclass + class InventoryItem: + name: Union[str, List[str]] = field(default_factory=list) + price: Union[float, List[float]] = field(default_factory=list) + stock: Union[int, List[int]] = field(default_factory=list) + + .. _topics-loaders-processors: Input and Output processors From 1335d9053e08c0321927728060d4eba0b6da687b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 22 Jun 2020 14:05:44 -0300 Subject: [PATCH 117/568] Update docs/topics/loaders.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves <adrian@chaves.io> --- docs/topics/loaders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index d70e03ad4..c5f121dbf 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -25,7 +25,7 @@ Using Item Loaders to populate items To use an Item Loader, you must first instantiate it. You can either instantiate it with an :ref:`item object <topics-items>` or without one, in which -case an instance of :class:`~scrapy.item.Item` is automatically created in the +case an :ref:`item object <topics-items>` is automatically created in the Item Loader ``__init__`` method using the :ref:`item <topics-items>` class specified in the :attr:`ItemLoader.default_item_class` attribute. From 73b6ce8cb560da71b082efdba44b90e1ea932b17 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Mon, 22 Jun 2020 14:13:37 -0300 Subject: [PATCH 118/568] Update docs about dataclass items and loaders --- docs/topics/loaders.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index c5f121dbf..e921395d2 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -84,9 +84,11 @@ Working with dataclass items By default, :ref:`dataclass items <dataclass-items>` require all fields to be passed when created. This could be an issue when using dataclass items with -item loaders, since fields could be populated incrementally. +item loaders: unless a pre-populated item is passed to the loader, fields +will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`, +:meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods. -Given the way that item loaders store data internally, the recommended approach +Given the way that item loaders store data internally, one approach to overcome this is to define items using the :func:`~dataclasses.field` function, with ``list`` as the ``default_factory`` argument:: From cfd039aeb6d46f7c120edd82ce06d6b9f4f8e7db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 22 Jun 2020 19:28:33 +0200 Subject: [PATCH 119/568] Remove a duplicate GCS_PROJECT_ID reference target --- docs/topics/media-pipeline.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 01de3dedb..096618648 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -204,7 +204,6 @@ For self-hosting you also might feel the need not to use SSL and not to verify S Google Cloud Storage --------------------- -.. setting:: GCS_PROJECT_ID .. setting:: FILES_STORE_GCS_ACL .. setting:: IMAGES_STORE_GCS_ACL From 3672f5f988cbb29c6bebb7ed535c0d5ad941d868 Mon Sep 17 00:00:00 2001 From: Lukas Anzinger <lukas@lukasanzinger.at> Date: Tue, 23 Jun 2020 14:51:21 +0200 Subject: [PATCH 120/568] Spider constructor expects name as argument, not start_urls. Fixes #4644 --- tests/test_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 930a5dd99..2b6cb0902 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -296,7 +296,7 @@ class StartUrlsSpider(Spider): def __init__(self, start_urls): self.start_urls = start_urls - super(StartUrlsSpider, self).__init__(start_urls) + super(StartUrlsSpider, self).__init__(name='StartUrlsSpider') def parse(self, response): pass From a97ac0adf86b67a16f09c41f618f736adb872503 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Wed, 24 Jun 2020 06:40:20 +0530 Subject: [PATCH 121/568] test: GET request for HTTP2Client using mockserver --- scrapy/core/http2/protocol.py | 4 +- scrapy/core/http2/stream.py | 10 +--- tests/test_http2_client_protocol.py | 71 ++++++++++------------------- 3 files changed, 30 insertions(+), 55 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 188c14c15..455d8777e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -9,6 +9,8 @@ from h2.events import ( DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) + +from twisted.internet.ssl import Certificate from twisted.internet.protocol import connectionDone, Protocol from scrapy.core.http2.stream import Stream, StreamCloseReason @@ -115,7 +117,7 @@ class H2ClientProtocol(Protocol): self.destination = self.transport.getPeer() LOGGER.info('Connection made to {}'.format(self.destination)) - self._metadata['certificate'] = self.transport.getPeerCertificate() + self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) self.conn.initiate_connection() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 07a4428c8..112ce5bcd 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -117,7 +117,7 @@ class Stream: self._deferred_response = Deferred() def __str__(self): - return "Stream(id={})".format(self.stream_id) + return "Stream(id={})".format(repr(self.stream_id)) __repr__ = __str__ @@ -299,9 +299,6 @@ class Stream: def close(self, reason: StreamCloseReason): """Based on the reason sent we will handle each case. """ - # TODO: In case of abruptly stream close - # Q1. Do we need to send the request again? - # Q2. What response should we send now? if self.stream_closed_server: raise StreamClosedError(self.stream_id) @@ -346,10 +343,7 @@ class Stream: and fires the response deferred callback with the generated response instance""" # TODO: - # 2. Should we fire this in case of - # 2.1 StreamReset in between when data is received partially - # 2.2 Forcefully closed the stream - # 3. Update Client Side Status Codes here + # 1. Update Client Side Status Codes here response_cls = responsetypes.from_args( headers=self._response['headers'], diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7830f7028..a67575d3c 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,71 +1,50 @@ -import os -import shutil +from urllib.parse import urlparse -from twisted.internet import defer, reactor +from twisted.internet import reactor from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint -from twisted.internet.ssl import optionsForClientTLS -from twisted.protocols.policies import WrappingFactory -from twisted.python.filepath import FilePath +from twisted.internet.ssl import CertificateOptions from twisted.trial import unittest -from twisted.web import static, server from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.http import Request -from tests.mockserver import ssl_context_factory +from scrapy.http import Request, Response +from tests.mockserver import MockServer class Http2ClientProtocolTestCase(unittest.TestCase): scheme = 'https' - # only used for HTTPS tests - file_key = 'keys/localhost.key' - file_certificate = 'keys/localhost.crt' - def setUp(self): # Start server for testing - self.path_temp = self.mktemp() - os.mkdir(self.path_temp) - FilePath(self.path_temp).child('file').setContent(b"0123456789") - r = static.File(self.path_temp) + self.mockserver = MockServer() + self.mockserver.__enter__() - self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) - self.host = 'localhost' - if self.scheme is 'https': - self.port = reactor.listenSSL( - 0, self.wrapper, - ssl_context_factory(self.file_key, self.file_certificate), - interface=self.host - ) + if self.scheme == 'https': + self.url = urlparse(self.mockserver.https_address) else: - self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) - - self.port_number = self.port.getHost().port - - # Connect to the server using the custom HTTP2ClientProtocol - options = optionsForClientTLS( - hostname=self.host, - acceptableProtocols=[b'h2'] - ) + self.url = urlparse(self.mockserver.http_address) self.protocol = H2ClientProtocol() - connectProtocol( - endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options), - protocol=self.protocol - ) + # Connect to the server using the custom HTTP2ClientProtocol + options = CertificateOptions(acceptableProtocols=[b'h2']) + endpoint = SSL4ClientEndpoint(reactor, self.url.hostname, self.url.port, options) + connectProtocol(endpoint, self.protocol) def getURL(self, path): - return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path) + return "{}://{}:{}/{}".format(self.url.scheme, self.url.hostname, self.url.port, path) - @defer.inlineCallbacks def tearDown(self): - yield self.port.stopListening() - shutil.rmtree(self.path_temp) + self.mockserver.__exit__(None, None, None) def test_download(self): - request = Request(self.getURL('file')) + request = Request(self.getURL('')) + + def assert_response(response: Response): + self.assertEqual(response.body, b'Scrapy mock HTTP server\n') + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + d = self.protocol.request(request) - d.addCallback(lambda response: response.body) - d.addCallback(self.assertEqual, b"0123456789") + d.addCallback(assert_response) return d From 69f6d038c0bc51a9de706890621d2ce183f79e09 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Wed, 24 Jun 2020 07:06:32 +0530 Subject: [PATCH 122/568] feat: TypedDict for Stream._response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - remove test_protocol.py as working testing environment is setup 🙂🙃 - Add typing_extensions as dependency to support TypedDict for python<3.8 --- scrapy/core/http2/protocol.py | 12 +----- scrapy/core/http2/stream.py | 11 ++++- scrapy/core/http2/test_protocol.py | 67 ------------------------------ setup.py | 4 +- 4 files changed, 15 insertions(+), 79 deletions(-) delete mode 100644 scrapy/core/http2/test_protocol.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 455d8777e..9b8ec6c77 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -9,9 +9,8 @@ from h2.events import ( DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) - -from twisted.internet.ssl import Certificate from twisted.internet.protocol import connectionDone, Protocol +from twisted.internet.ssl import Certificate from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.http import Request @@ -22,12 +21,7 @@ LOGGER = logging.getLogger(__name__) class H2ClientProtocol(Protocol): # TODO: # 1. Check for user-agent while testing - # 2. Add support for cookies - # 3. Handle priority updates (Not required) - # 4. Handle case when received events have StreamID = 0 (applied to H2Connection) - # 1 & 2: - # - Automatically handled by the Request middleware - # - request.headers will have 'Set-Cookie' value + # 2. Handle case when received events have StreamID = 0 (applied to H2Connection) def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') @@ -185,8 +179,6 @@ class H2ClientProtocol(Protocol): self.streams[stream_id].close(StreamCloseReason.ENDED) def stream_reset(self, event: StreamReset): - # TODO: event.stream_id was abruptly closed - # Q. What should be the response? (Failure/Partial/???) self.streams[event.stream_id].close(StreamCloseReason.RESET) def window_updated(self, event: WindowUpdated): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 112ce5bcd..023f4f4eb 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -10,11 +10,20 @@ from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred, CancelledError from twisted.python.failure import Failure from twisted.web.client import ResponseFailed +# for python < 3.8 -- typing.TypedDict is undefined +from typing_extensions import TypedDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes + +class _ResponseTypedDict(TypedDict): + body: BytesIO + flow_controlled_size: int + headers: Headers + + LOGGER = logging.getLogger(__name__) @@ -100,7 +109,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response = { + self._response: _ResponseTypedDict = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. 'body': BytesIO(), diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.py deleted file mode 100644 index c7782a518..000000000 --- a/scrapy/core/http2/test_protocol.py +++ /dev/null @@ -1,67 +0,0 @@ -# This is simple script to test - -import json - -from twisted.internet import reactor -from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint -from twisted.internet.ssl import optionsForClientTLS - -from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.http import Request, Response, JsonRequest - -try: - with open('data.json', 'r') as f: - JSON_DATA = json.load(f) -except: - JSON_DATA = { - "data": "To test for really large amount of data -- Add data.json with lots of data.", - "why": "To test whether correct data is sent :)" - } - -# Use nghttp2 for testing whether basic setup works - for small response -HTTPBIN_AUTHORITY = u'nghttp2.org' -HTTPBIN_REQUEST_URLS = 1 * [ - Request(url='https://nghttp2.org/httpbin/get', method='GET'), - Request(url='https://nghttp2.org/httpbin/post', method='POST'), - JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA), -] - -# Use POKE_API for testing large responses -POKE_API_AUTHORITY = u'pokeapi.co' -POKE_API_REQUESTS = 15 * [ - Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'), - Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'), - Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'), - Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'), # should give 404 -] - -AUTHORITY = POKE_API_AUTHORITY -REQUEST_URLS = POKE_API_REQUESTS - -options = optionsForClientTLS( - hostname=AUTHORITY, - acceptableProtocols=[b'h2'], -) - -protocol = H2ClientProtocol() - -count_responses = 1 - - -def print_response(response): - global count_responses - assert isinstance(response, Response) - print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body))) - count_responses = count_responses + 1 - - -for request in REQUEST_URLS: - d = protocol.request(request) - d.addCallback(print_response) - -connectProtocol( - SSL4ClientEndpoint(reactor, AUTHORITY, 443, options), - protocol -) - -reactor.run() diff --git a/setup.py b/setup.py index dafa5684a..d1470df5e 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=3.5', + python_requires='>=3.5.2', install_requires=[ 'Twisted>=17.9.0', 'Twisted[http2]>=17.9.0' @@ -80,6 +80,8 @@ setup( 'w3lib>=1.17.0', 'zope.interface>=4.1.3', 'protego>=0.1.15', + 'itemadapter>=0.1.0', + 'typing_extensions>=3.7' ], extras_require=extras_require, ) From 6f4c964aa4949584cfd251cf2aa2c0c7de6cf251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 24 Jun 2020 12:26:38 +0200 Subject: [PATCH 123/568] Cover Scrapy 2.2.0 in the release notes (#4630) --- docs/contributing.rst | 5 +- docs/news.rst | 195 +++++++++++++++++++++++++++++++ docs/topics/media-pipeline.rst | 9 +- docs/topics/request-response.rst | 4 +- scrapy/http/response/text.py | 2 + scrapy/utils/misc.py | 5 +- 6 files changed, 214 insertions(+), 6 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index aed5ab92e..7b901dd00 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -155,6 +155,9 @@ Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports removal, etc) in separate commits from functional changes. This will make pull requests easier to review and more likely to get merged. + +.. _coding-style: + Coding style ============ @@ -163,7 +166,7 @@ Scrapy: * Unless otherwise specified, follow :pep:`8`. -* It's OK to use lines longer than 80 chars if it improves the code +* It's OK to use lines longer than 79 chars if it improves the code readability. * Don't put your name in the code you contribute; git provides enough diff --git a/docs/news.rst b/docs/news.rst index a158246eb..80d130e4a 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,201 @@ Release notes ============= +.. _release-2.2.0: + +Scrapy 2.2.0 (2020-06-24) +------------------------- + +Highlights: + +* Python 3.5.2+ is required now +* :ref:`dataclass objects <dataclass-items>` and + :ref:`attrs objects <attrs-items>` are now valid :ref:`item types + <item-types>` +* New :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method +* New :signal:`bytes_received` signal that allows canceling response download +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` fixes + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Support for Python 3.5.0 and 3.5.1 has been dropped; Scrapy now refuses to + run with a Python version lower than 3.5.2, which introduced + :class:`typing.Type` (:issue:`4615`) + + +Deprecations +~~~~~~~~~~~~ + +* :meth:`TextResponse.body_as_unicode + <scrapy.http.TextResponse.body_as_unicode>` is now deprecated, use + :attr:`TextResponse.text <scrapy.http.TextResponse.text>` instead + (:issue:`4546`, :issue:`4555`, :issue:`4579`) + +* :class:`scrapy.item.BaseItem` is now deprecated, use + :class:`scrapy.item.Item` instead (:issue:`4534`) + + +New features +~~~~~~~~~~~~ + +* :ref:`dataclass objects <dataclass-items>` and + :ref:`attrs objects <attrs-items>` are now valid :ref:`item types + <item-types>`, and a new itemadapter_ library makes it easy to + write code that :ref:`supports any item type <supporting-item-types>` + (:issue:`2749`, :issue:`2807`, :issue:`3761`, :issue:`3881`, :issue:`4642`) + +* A new :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method + allows to deserialize JSON responses (:issue:`2444`, :issue:`4460`, + :issue:`4574`) + +* A new :signal:`bytes_received` signal allows monitoring response download + progress and :ref:`stopping downloads <topics-stop-response-download>` + (:issue:`4205`, :issue:`4559`) + +* The dictionaries in the result list of a :ref:`media pipeline + <topics-media-pipeline>` now include a new key, ``status``, which indicates + if the file was downloaded or, if the file was not downloaded, why it was + not downloaded; see :meth:`FilesPipeline.get_media_requests + <scrapy.pipelines.files.FilesPipeline.get_media_requests>` for more + information (:issue:`2893`, :issue:`4486`) + +* When using :ref:`Google Cloud Storage <media-pipeline-gcs>` for + a :ref:`media pipeline <topics-media-pipeline>`, a warning is now logged if + the configured credentials do not grant the required permissions + (:issue:`4346`, :issue:`4508`) + +* :ref:`Link extractors <topics-link-extractors>` are now serializable, + as long as you do not use :ref:`lambdas <lambda>` for parameters; for + example, you can now pass link extractors in :attr:`Request.cb_kwargs + <scrapy.http.Request.cb_kwargs>` or + :attr:`Request.meta <scrapy.http.Request.meta>` when :ref:`persisting + scheduled requests <topics-jobs>` (:issue:`4554`) + +* Upgraded the :ref:`pickle protocol <pickle-protocols>` that Scrapy uses + from protocol 2 to protocol 4, improving serialization capabilities and + performance (:issue:`4135`, :issue:`4541`) + +* :func:`scrapy.utils.misc.create_instance` now raises a :exc:`TypeError` + exception if the resulting instance is ``None`` (:issue:`4528`, + :issue:`4532`) + +.. _itemadapter: https://github.com/scrapy/itemadapter + + +Bug fixes +~~~~~~~~~ + +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer + discards cookies defined in :attr:`Request.headers + <scrapy.http.Request.headers>` (:issue:`1992`, :issue:`2400`) + +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer + re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter + of the ``__init__`` method of :class:`~scrapy.http.Request` + (:issue:`2400`, :issue:`3575`) + +* When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is + ``False`` and the crawl yields no items, Scrapy no longer stops feed + exports after the first URI (:issue:`4621`, :issue:`4626`) + +* :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine + syntax <topics/coroutines>` no longer need to return an iterable, and may + instead return a :class:`~scrapy.http.Request` object, an + :ref:`item <topics-items>`, or ``None`` (:issue:`4609`) + +* The :command:`startproject` command now ensures that the generated project + folders and files have the right permissions (:issue:`4604`) + +* Fix a :exc:`KeyError` exception being sometimes raised from + :class:`scrapy.utils.datatypes.LocalWeakReferencedCache` (:issue:`4597`, + :issue:`4599`) + +* When :setting:`FEEDS` defines multiple URIs, log messages about items being + stored now contain information from the corresponding feed, instead of + always containing information about only one of the feeds (:issue:`4619`, + :issue:`4629`) + + +Documentation +~~~~~~~~~~~~~ + +* Added a new section about :ref:`accessing cb_kwargs from errbacks + <errback-cb_kwargs>` (:issue:`4598`, :issue:`4634`) + +* Covered chompjs_ in :ref:`topics-parsing-javascript` (:issue:`4556`, + :issue:`4562`) + +* Removed from :doc:`topics/coroutines` the warning about the API being + experimental (:issue:`4511`, :issue:`4513`) + +* Removed references to unsupported versions of :doc:`Twisted + <twisted:index>` (:issue:`4533`) + +* Updated the description of the :ref:`screenshot pipeline example + <ScreenshotPipeline>`, which now uses :doc:`coroutine syntax + <topics/coroutines>` instead of returning a + :class:`~twisted.internet.defer.Deferred` (:issue:`4514`, :issue:`4593`) + +* Removed a misleading import line from the + :func:`scrapy.utils.log.configure_logging` code example (:issue:`4510`, + :issue:`4587`) + +* The display-on-hover behavior of internal documentation references now also + covers links to :ref:`commands <topics-commands>`, :attr:`Request.meta + <scrapy.http.Request.meta>` keys, :ref:`settings <topics-settings>` and + :ref:`signals <topics-signals>` (:issue:`4495`, :issue:`4563`) + +* It is again possible to download the documentation for offline reading + (:issue:`4578`, :issue:`4585`) + +* Removed backslashes preceding ``*args`` and ``**kwargs`` in some function + and method signatures (:issue:`4592`, :issue:`4596`) + +.. _chompjs: https://github.com/Nykakin/chompjs + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Adjusted the code base further to our :ref:`style guidelines + <coding-style>` (:issue:`4237`, :issue:`4525`, :issue:`4538`, + :issue:`4539`, :issue:`4540`, :issue:`4542`, :issue:`4543`, :issue:`4544`, + :issue:`4545`, :issue:`4557`, :issue:`4558`, :issue:`4566`, :issue:`4568`, + :issue:`4572`) + +* Removed remnants of Python 2 support (:issue:`4550`, :issue:`4553`, + :issue:`4568`) + +* Improved code sharing between the :command:`crawl` and :command:`runspider` + commands (:issue:`4548`, :issue:`4552`) + +* Replaced ``chain(*iterable)`` with ``chain.from_iterable(iterable)`` + (:issue:`4635`) + +* You may now run the :mod:`asyncio` tests with Tox on any Python version + (:issue:`4521`) + +* Updated test requirements to reflect an incompatibility with pytest 5.4 and + 5.4.1 (:issue:`4588`) + +* Improved :class:`~scrapy.spiderloader.SpiderLoader` test coverage for + scenarios involving duplicate spider names (:issue:`4549`, :issue:`4560`) + +* Configured Travis CI to also run the tests with Python 3.5.2 + (:issue:`4518`, :issue:`4615`) + +* Added a `Pylint <https://www.pylint.org/>`_ job to Travis CI + (:issue:`3727`) + +* Added a `Mypy <http://mypy-lang.org/>`_ job to Travis CI (:issue:`4637`) + +* Made use of set literals in tests (:issue:`4573`) + +* Cleaned up the Travis CI configuration (:issue:`4517`, :issue:`4519`, + :issue:`4522`, :issue:`4537`) + + .. _release-2.1.0: Scrapy 2.1.0 (2020-04-24) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 01de3dedb..9f2a06dd7 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -201,6 +201,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S .. _s3.scality: https://s3.scality.com/ .. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + +.. _media-pipeline-gcs: + Google Cloud Storage --------------------- @@ -475,7 +478,11 @@ See here the methods that you can override in your custom Files Pipeline: * ``checksum`` - a `MD5 hash`_ of the image contents - * ``status`` - the file status indication. It can be one of the following: + * ``status`` - the file status indication. + + .. versionadded:: 2.2 + + It can be one of the following: * ``downloaded`` - file was downloaded. * ``uptodate`` - file was not downloaded, as it was downloaded recently, diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d88d40b00..fbd8e4b73 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -191,7 +191,7 @@ Request objects In case of a failure to process the request, this dict can be accessed as ``failure.request.cb_kwargs`` in the request's errback. For more information, - see :ref:`topics-request-response-ref-accessing-callback-arguments-in-errback`. + see :ref:`errback-cb_kwargs`. .. method:: Request.copy() @@ -316,7 +316,7 @@ errors if needed:: request = failure.request self.logger.error('TimeoutError on %s', request.url) -.. _topics-request-response-ref-accessing-callback-arguments-in-errback: +.. _errback-cb_kwargs: Accessing additional data in errback functions ---------------------------------------------- diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 40cf3f483..b43fe5c19 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -74,6 +74,8 @@ class TextResponse(Response): def json(self): """ + .. versionadded:: 2.2 + Deserialize a JSON document to a Python object. """ if self._cached_decoded_json is _NONE: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index a7808cb2c..8e5fde246 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -138,8 +138,9 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``. - Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an - extension has not been implemented correctly). + .. versionchanged:: 2.2 + Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an + extension has not been implemented correctly). """ if settings is None: if crawler is None: From 9f60481360628bafe467f59b2144d69204b8b4e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 24 Jun 2020 12:27:39 +0200 Subject: [PATCH 124/568] =?UTF-8?q?Bump=20version:=202.1.0=20=E2=86=92=202?= =?UTF-8?q?.2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index de22a2783..8d4d74bc5 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.1.0 +current_version = 2.2.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 7ec1d6db4..ccbccc3dc 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.1.0 +2.2.0 From c3cee74fd401e6a6307b5eb1786e532bb2cd5aa8 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Fri, 26 Jun 2020 18:45:21 +0300 Subject: [PATCH 125/568] Change default value of FEED_STORAGE_BATCH_ITEM_COUNT to 0 --- docs/topics/feed-exports.rst | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 20 ++++++++++---------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 866ce78eb..0b37e9a7d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -435,7 +435,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter FEED_STORAGE_BATCH_ITEM_COUNT ----------------------------- -Default: ``None`` +Default: ``0`` If assigned an integer number higher than ``0``, Scrapy generates multiple output files storing up to the specified number of items in each output file. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 1331782e3..e06116acd 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -243,7 +243,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) + self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5a7dc533e..810acd5a3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_ITEM_COUNT = None +FEED_STORAGE_BATCH_ITEM_COUNT = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1a6a5624b..578cd396b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -1262,7 +1262,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'FEED_STORAGE_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, settings=settings) + yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """ @@ -1412,14 +1412,14 @@ class BatchDeliveriesTest(FeedExportTestBase): bucket_name=s3_test_bucket_name, prefix=prefix ) storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) - settings = { + settings = Settings({ 'FEEDS': { s3_test_file_uri: { 'format': 'json', }, }, 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, - } + }) items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), @@ -1436,7 +1436,7 @@ class BatchDeliveriesTest(FeedExportTestBase): s3 = boto3.resource('s3') my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') with MockServer() as s: runner = CrawlerRunner(Settings(settings)) From 88a52198b90faa0129c8e05072197cdffbb9653b Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Sat, 27 Jun 2020 11:50:26 +0300 Subject: [PATCH 126/568] Add batch_item_count support in FEEDS setting --- scrapy/extensions/feedexport.py | 5 +++-- tests/test_feedexport.py | 39 ++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e06116acd..2312c994e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values - logger = logging.getLogger(__name__) @@ -337,7 +336,9 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one - if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count: + if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \ + and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count', + self.storage_batch_item_count): uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 578cd396b..3bc0c083c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1327,8 +1327,9 @@ class BatchDeliveriesTest(FeedExportTestBase): '<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>' ).encode('latin-1') ], - 'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), - 'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')], + 'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'), + 'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')], + 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], } settings = { @@ -1348,9 +1349,16 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { 'format': 'csv', 'indent': None, - 'fields': ['bar', 'foo'], + 'fields': ['foo', 'bar'], 'encoding': 'utf-8', }, + os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { + 'format': 'jsonlines', + 'indent': None, + 'fields': ['foo', 'bar'], + 'encoding': 'utf-8', + 'batch_item_count': 0, + }, }, 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } @@ -1359,6 +1367,31 @@ class BatchDeliveriesTest(FeedExportTestBase): for expected_batch, got_batch in zip(expected, data[fmt]): self.assertEqual(expected_batch, got_batch) + @defer.inlineCallbacks + def test_batch_item_count_feeds_setting(self): + items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] + + formats = { + 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'), + '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], + } + + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { + 'format': 'jsonlines', + 'indent': None, + 'fields': ['foo', 'bar'], + 'encoding': 'utf-8', + 'batch_item_count': 1, + }, + }, + } + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt]): + self.assertEqual(expected_batch, got_batch) + @defer.inlineCallbacks def test_batch_path_differ(self): """ From 23da8e106822cf2805e8d74c382a31bca70986f3 Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Sat, 27 Jun 2020 20:36:45 +0530 Subject: [PATCH 127/568] Add schemaless http proxy support --- scrapy/core/downloader/handlers/http11.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 22c9ac520..73e56c87d 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -7,7 +7,7 @@ import warnings from contextlib import suppress from io import BytesIO from time import time -from urllib.parse import urldefrag +from urllib.parse import urldefrag, urlparse from twisted.internet import defer, protocol, ssl from twisted.internet.endpoints import TCP4ClientEndpoint @@ -255,7 +255,7 @@ class ScrapyProxyAgent(Agent): bindAddress=bindAddress, pool=pool, ) - self._proxyURI = URI.fromBytes(proxyURI) + self._proxyURI = URI.fromBytes(urlparse(proxyURI)._replace(scheme=b'http').geturl()) def request(self, method, uri, headers=None, bodyProducer=None): """ From f53f06020b740b450c7555d562565a27918e8036 Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Sat, 27 Jun 2020 23:28:40 +0530 Subject: [PATCH 128/568] Test http schemaless proxy --- tests/test_downloader_handlers.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 51deb20f4..5854659dd 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -131,6 +131,7 @@ class ContentLengthHeaderResource(resource.Resource): A testing resource which renders itself as the value of the Content-Length header from the request. """ + def render(self, request): return request.requestHeaders.getRawHeaders(b"content-length")[0] @@ -186,6 +187,7 @@ class EmptyContentTypeHeaderResource(resource.Resource): A testing resource which renders itself as the value of request body without content-type header in response. """ + def render(self, request): request.setHeader("content-type", "") return request.content.read() @@ -733,6 +735,16 @@ class Http11ProxyTestCase(HttpProxyTestCase): timeout = yield self.assertFailure(d, error.TimeoutError) self.assertIn(domain, timeout.osError) + def test_download_with_proxy_without_http_scheme(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'http://example.com') + + http_proxy = self.getURL('').replace('http:', '') + request = Request('http://example.com', meta={'proxy': http_proxy}) + return self.download_request(request, Spider('foo')).addCallback(_test) + class HttpDownloadHandlerMock: From 05c2587c6a32b84a94463f2b1187e49f94957aa2 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Sun, 28 Jun 2020 09:45:45 +0300 Subject: [PATCH 129/568] Docs update and tiny fixes --- docs/topics/feed-exports.rst | 1 + tests/test_feedexport.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0b37e9a7d..3da56821e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -272,6 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition. * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` +* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` .. setting:: FEED_EXPORT_ENCODING diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 3bc0c083c..542cce70f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'fields': ['foo', 'bar'], 'encoding': 'utf-8', }, - os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { + os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { 'format': 'jsonlines', 'indent': None, 'fields': ['foo', 'bar'], @@ -1423,8 +1423,8 @@ class BatchDeliveriesTest(FeedExportTestBase): [testenv] setenv = AWS_SECRET_ACCESS_KEY = ABCD - AWS_ACCESS_KEY_ID = ABCD - S3_TEST_BUCKET_NAME = ABCD + AWS_ACCESS_KEY_ID = EFGH + S3_TEST_BUCKET_NAME = IJKL """ try: import boto3 From 690dd7f38bfd245428bdc618dcca1fbc26284df1 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Sun, 28 Jun 2020 16:35:04 +0530 Subject: [PATCH 130/568] test: GET & POST request test for h2 client - Remove repeated dependency Twisted from setup.py - Test for both GET & POST when - Only 1 request - Large number (=20) of requests and - Small Data (10 KB) per request - Large Data (10 MB) per request - Test when request is cancelled by the client' BREAKING CHANGES Tests raises OpenSSL.SSL.Error when run using tox. However, all tests passes when ran using `python -m unittest`. --- scrapy/core/http2/protocol.py | 4 +- scrapy/core/http2/stream.py | 66 +++--- setup.py | 1 - tests/test_http2_client_protocol.py | 321 +++++++++++++++++++++++++--- 4 files changed, 331 insertions(+), 61 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 9b8ec6c77..7fb935f10 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -129,11 +129,11 @@ class H2ClientProtocol(Protocol): """ # Pop all streams which were pending and were not yet started for stream_id in list(self.streams): - self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST) + self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST, reason) self.conn.close_connection() - LOGGER.info("Connection lost with reason " + str(reason)) + LOGGER.warning("Connection lost with reason " + str(reason)) def _handle_events(self, events): """Private method which acts as a bridge between the events diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 023f4f4eb..a26a33918 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -19,8 +19,15 @@ from scrapy.responsetypes import responsetypes class _ResponseTypedDict(TypedDict): + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. body: BytesIO + + # The amount of data received that counts against the flow control + # window flow_controlled_size: int + + # Headers received after sending the request headers: Headers @@ -40,6 +47,9 @@ class StreamCloseReason(IntFlag): # Expected response body size is more than allowed limit MAXSIZE_EXCEEDED = auto() + # When the response deferred is cancelled + CANCELLED = auto() + class Stream: """Represents a single HTTP/2 Stream. @@ -110,20 +120,16 @@ class Stream: # this response is then converted to appropriate Response class # passed to the response deferred callback self._response: _ResponseTypedDict = { - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. 'body': BytesIO(), - - # The amount of data received that counts against the flow control - # window 'flow_controlled_size': 0, - - # Headers received after sending the request 'headers': Headers({}) } - # TODO: Add canceller for the Deferred below - self._deferred_response = Deferred() + def _cancel(_): + # Close this stream as gracefully as possible :) + self.reset_stream(StreamCloseReason.CANCELLED) + + self._deferred_response = Deferred(_cancel) def __str__(self): return "Stream(id={})".format(repr(self.stream_id)) @@ -177,13 +183,6 @@ class Stream: and has initiated request already by sending HEADER frame. If not then stream will raise ProtocolError (raise by h2 state machine). """ - # TODO: - # 1. Add test for sending very large data - # 2. Add test for small data - # 3. Both (1) and (2) should be tested for - # 3.1 Large number of request - # 3.2 Small number of requests - if self.stream_closed_local: raise StreamClosedError(self.stream_id) @@ -221,7 +220,6 @@ class Stream: # End the stream if no more data needs to be send if self.remaining_content_length == 0: - self.stream_closed_local = True self._conn.end_stream(self.stream_id) # Write data to transport -- Empty the outstanding data @@ -288,7 +286,6 @@ class Stream: def reset_stream(self, reason=StreamCloseReason.RESET): """Close this stream by sending a RST_FRAME to the remote peer""" - # TODO: Q. REFUSED_STREAM or CANCEL ? if self.stream_closed_local: raise StreamClosedError(self.stream_id) @@ -305,14 +302,16 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason): + def close(self, reason: StreamCloseReason, failure=None): """Based on the reason sent we will handle each case. """ if self.stream_closed_server: raise StreamClosedError(self.stream_id) + self._cb_close(self.stream_id) self.stream_closed_server = True + # Do nothing if the response deferred was cancelled flags = None if b'Content-Length' not in self._response['headers']: # Missing Content-Length - PotentialDataLoss @@ -320,7 +319,6 @@ class Stream: elif self._is_data_lost(): if self._fail_on_dataloss: self._deferred_response.errback(ResponseFailed([Failure()])) - self._cb_close(self.stream_id) return else: flags = ['dataloss'] @@ -328,10 +326,19 @@ class Stream: if reason is StreamCloseReason.ENDED: self._fire_response_deferred(flags) - elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST): - # Stream was abruptly ended here - self._deferred_response.errback(ResponseFailed([Failure()])) + # Stream was abruptly ended here + elif reason is StreamCloseReason.CANCELLED: + # Client has cancelled the request. Remove all the data + # received and fire the response deferred with no flags set + self._response['body'].truncate(0) + self._response['headers'].clear() + self._fire_response_deferred() + elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST): + if failure is None: + self._deferred_response.errback(ResponseFailed([Failure()])) + else: + self._deferred_response.errback(failure) elif reason is StreamCloseReason.MAXSIZE_EXCEEDED: expected_size = int(self._response['headers'].get(b'Content-Length', -1)) error_msg = ("Cancelling download of {url}: expected response " @@ -345,22 +352,21 @@ class Stream: LOGGER.error(error_msg, error_args) self._deferred_response.errback(CancelledError(error_msg.format(**error_args))) - self._cb_close(self.stream_id) - def _fire_response_deferred(self, flags=None): """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" - # TODO: - # 1. Update Client Side Status Codes here + # TODO: Update Client Side Status Codes here + body = self._response['body'].getvalue() response_cls = responsetypes.from_args( headers=self._response['headers'], url=self._request.url, - body=self._response['body'] + body=body ) - # If there is no :status in headers then + # If there is no :status in headers + # (happens when client called response_deferred.cancel()) # HTTP Status Code: 499 - Client Closed Request status = self._response['headers'].get(':status', '499') @@ -368,7 +374,7 @@ class Stream: url=self._request.url, status=status, headers=self._response['headers'], - body=self._response['body'].getvalue(), + body=body, request=self._request, flags=flags, certificate=self._conn_metadata['certificate'], diff --git a/setup.py b/setup.py index d1470df5e..575c74e7f 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,6 @@ setup( ], python_requires='>=3.5.2', install_requires=[ - 'Twisted>=17.9.0', 'Twisted[http2]>=17.9.0' 'cryptography>=2.0', 'cssselect>=0.9.1', diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index a67575d3c..0f3730c9e 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,50 +1,315 @@ -from urllib.parse import urlparse +# TODO: Add test cases for +# 1. No Content Length response header +# 2. Cancel Response Deferred +import json +import os +import random +import shutil +import string from twisted.internet import reactor -from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint -from twisted.internet.ssl import CertificateOptions -from twisted.trial import unittest +from twisted.internet.defer import inlineCallbacks, DeferredList +from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint +from twisted.internet.protocol import Factory +from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate +from twisted.trial.unittest import TestCase +from twisted.web.http import Request as TxRequest +from twisted.web.resource import Resource +from twisted.web.server import Site +from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.http import Request, Response -from tests.mockserver import MockServer +from scrapy.http import Request, Response, JsonRequest +from tests.mockserver import ssl_context_factory -class Http2ClientProtocolTestCase(unittest.TestCase): +def generate_random_string(size): + return ''.join(random.choices( + string.ascii_uppercase + string.digits, + k=size + )) + + +def make_html_body(val): + response = '''<html> +<h1>Hello from HTTP2<h1> +<p>{}</p> +</html>'''.format(val) + return bytes(response, 'utf-8') + + +class Data: + SMALL_SIZE = 1024 * 10 # 10 KB + LARGE_SIZE = (1024 ** 2) * 10 # 10 MB + + STR_SMALL = generate_random_string(SMALL_SIZE) + STR_LARGE = generate_random_string(LARGE_SIZE) + + EXTRA_SMALL = generate_random_string(1024 * 15) + EXTRA_LARGE = generate_random_string((1024 ** 2) * 15) + + HTML_SMALL = make_html_body(STR_SMALL) + HTML_LARGE = make_html_body(STR_LARGE) + + JSON_SMALL = {'data': STR_SMALL} + JSON_LARGE = {'data': STR_LARGE} + + +class LeafResource(Resource): + isLeaf = True + + +class GetDataHtmlSmall(LeafResource): + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'text/html; charset=UTF-8') + return Data.HTML_SMALL + + +class GetDataHtmlLarge(LeafResource): + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'text/html; charset=UTF-8') + return Data.HTML_LARGE + + +class PostDataJsonMixin: + @staticmethod + def make_response(request: TxRequest, extra_data: str): + response = { + 'request-headers': {}, + 'request-body': json.loads(request.content.read()), + 'extra-data': extra_data + } + for k, v in request.requestHeaders.getAllRawHeaders(): + response['request-headers'][k.decode('utf-8')] = v[0].decode('utf-8') + + response_bytes = bytes(json.dumps(response), 'utf-8') + request.setHeader('Content-Type', 'application/json') + return response_bytes + + +class PostDataJsonSmall(LeafResource, PostDataJsonMixin): + def render_POST(self, request: TxRequest): + return self.make_response(request, Data.EXTRA_SMALL) + + +class PostDataJsonLarge(LeafResource, PostDataJsonMixin): + def render_POST(self, request: TxRequest): + return self.make_response(request, Data.EXTRA_LARGE) + + +def get_client_certificate(key_file, certificate_file): + with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: + pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) + + return PrivateCertificate.loadPEM(pem) + + +class Https2ClientProtocolTestCase(TestCase): scheme = 'https' + key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key') + certificate_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.crt') + def _init_resource(self): + self.temp_directory = self.mktemp() + os.mkdir(self.temp_directory) + r = File(self.temp_directory) + r.putChild(b'get-data-html-small', GetDataHtmlSmall()) + r.putChild(b'get-data-html-large', GetDataHtmlLarge()) + + r.putChild(b'post-data-json-small', PostDataJsonSmall()) + r.putChild(b'post-data-json-large', PostDataJsonLarge()) + return r + + @inlineCallbacks def setUp(self): + # Initialize resource tree + root = self._init_resource() + self.site = Site(root, timeout=None) + # Start server for testing - self.mockserver = MockServer() - self.mockserver.__enter__() - + self.hostname = u'localhost' if self.scheme == 'https': - self.url = urlparse(self.mockserver.https_address) + context_factory = ssl_context_factory(self.key_file, self.certificate_file) + server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) else: - self.url = urlparse(self.mockserver.http_address) + server_endpoint = TCP4ServerEndpoint(reactor, 0, interface=self.hostname) + self.server = yield server_endpoint.listen(self.site) + self.port_number = self.server.getHost().port - self.protocol = H2ClientProtocol() - - # Connect to the server using the custom HTTP2ClientProtocol - options = CertificateOptions(acceptableProtocols=[b'h2']) - endpoint = SSL4ClientEndpoint(reactor, self.url.hostname, self.url.port, options) - connectProtocol(endpoint, self.protocol) - - def getURL(self, path): - return "{}://{}:{}/{}".format(self.url.scheme, self.url.hostname, self.url.port, path) + # Connect H2 client with server + client_certificate = get_client_certificate(self.key_file, self.certificate_file) + client_options = optionsForClientTLS( + hostname=self.hostname, + trustRoot=client_certificate, + acceptableProtocols=[b'h2'] + ) + h2_client_factory = Factory.forProtocol(H2ClientProtocol) + client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) + self.client = yield client_endpoint.connect(h2_client_factory) + @inlineCallbacks def tearDown(self): - self.mockserver.__exit__(None, None, None) + yield self.client.transport.loseConnection() + yield self.client.transport.abortConnection() + yield self.server.stopListening() + shutil.rmtree(self.temp_directory) - def test_download(self): - request = Request(self.getURL('')) + def get_url(self, path): + """ + :param path: Should have / at the starting compulsorily if not empty + :return: Complete url + """ + assert len(path) > 0 and (path[0] == '/' or path[0] == '&') + return "{}://{}:{}{}".format(self.scheme, self.hostname, self.port_number, path) - def assert_response(response: Response): - self.assertEqual(response.body, b'Scrapy mock HTTP server\n') - self.assertEqual(response.status, 200) + @staticmethod + def _check_repeat(get_deferred, count): + d_list = [] + for _ in range(count): + d = get_deferred() + d_list.append(d) + + return DeferredList(d_list, fireOnOneErrback=True) + + def _check_GET( + self, + request: Request, + expected_body, + expected_status + ): + def check_response(response: Response): + self.assertEqual(response.status, expected_status) + self.assertEqual(response.body, expected_body) self.assertEqual(response.request, request) self.assertEqual(response.url, request.url) - d = self.protocol.request(request) + content_length = int(response.headers.get('Content-Length')) + self.assertEqual(len(response.body), content_length) + + d = self.client.request(request) + d.addCallback(check_response) + d.addErrback(self.fail) + return d + + def test_GET_small_body(self): + request = Request(self.get_url('/get-data-html-small')) + return self._check_GET(request, Data.HTML_SMALL, 200) + + def test_GET_large_body(self): + request = Request(self.get_url('/get-data-html-large')) + return self._check_GET(request, Data.HTML_LARGE, 200) + + def _check_GET_x20(self, *args, **kwargs): + def get_deferred(): + return self._check_GET(*args, **kwargs) + + return self._check_repeat(get_deferred, 20) + + def test_GET_small_body_x20(self): + return self._check_GET_x20( + Request(self.get_url('/get-data-html-small')), + Data.HTML_SMALL, + 200 + ) + + def test_GET_large_body_x20(self): + return self._check_GET_x20( + Request(self.get_url('/get-data-html-large')), + Data.HTML_LARGE, + 200 + ) + + def _check_POST_json( + self, + request: Request, + expected_request_body, + expected_extra_data, + expected_status: int + ): + d = self.client.request(request) + + def assert_response(response: Response): + self.assertEqual(response.status, expected_status) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + + content_length = int(response.headers.get('Content-Length')) + self.assertEqual(len(response.body), content_length) + + # Parse the body + body = json.loads(response.body.decode('utf-8')) + self.assertIn('request-body', body) + self.assertIn('extra-data', body) + self.assertIn('request-headers', body) + + request_body = body['request-body'] + self.assertEqual(request_body, expected_request_body) + + extra_data = body['extra-data'] + self.assertEqual(extra_data, expected_extra_data) + + # Check if headers were sent successfully + request_headers = body['request-headers'] + for k, v in request.headers.items(): + k_str = k.decode('utf-8') + self.assertIn(k_str, request_headers) + self.assertEqual(request_headers[k_str], v[0].decode('utf-8')) + d.addCallback(assert_response) return d + + def test_POST_small_json(self): + request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) + return self._check_POST_json( + request, + Data.JSON_SMALL, + Data.EXTRA_SMALL, + 200 + ) + + def test_POST_large_json(self): + request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) + return self._check_POST_json( + request, + Data.JSON_LARGE, + Data.EXTRA_LARGE, + 200 + ) + + def _check_POST_json_x20(self, *args, **kwargs): + def get_deferred(): + return self._check_POST_json(*args, **kwargs) + + return self._check_repeat(get_deferred, 20) + + def test_POST_small_json_x20(self): + request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) + return self._check_POST_json_x20( + request, + Data.JSON_SMALL, + Data.EXTRA_SMALL, + 200 + ) + + def test_POST_large_json_x20(self): + request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) + return self._check_POST_json_x20( + request, + Data.JSON_LARGE, + Data.EXTRA_LARGE, + 200 + ) + + def test_cancel_request(self): + request = Request(url=self.get_url('/get-data-html-large')) + + def assert_response(response: Response): + self.assertEqual(response.status, 499) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + + d = self.client.request(request) + d.addCallback(assert_response) + d.cancel() + + return d From 6387445ef519124f393a20657e66383340fb1677 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Sun, 28 Jun 2020 18:44:57 +0530 Subject: [PATCH 131/568] test(tox.ini): change Twisted -> Twisted[http2] --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 27d21ade2..ada211b3c 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,7 @@ deps = pyOpenSSL==16.2.0 queuelib==1.4.2 service_identity==16.0.0 - Twisted==17.9.0 + Twisted[http2]==17.9.0 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt From 464f24f8c160466f93eb8ebb4ec8b84d1824eaa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com> Date: Mon, 29 Jun 2020 14:20:29 +0200 Subject: [PATCH 132/568] Add --data-raw to utils.curl and fix missing method with data (#4612) --- scrapy/utils/curl.py | 12 +++++++++--- tests/test_utils_curl.py | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 16639356e..67b22dbc5 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -17,8 +17,8 @@ class CurlParser(argparse.ArgumentParser): curl_parser = CurlParser() curl_parser.add_argument('url') curl_parser.add_argument('-H', '--header', dest='headers', action='append') -curl_parser.add_argument('-X', '--request', dest='method', default='get') -curl_parser.add_argument('-d', '--data', dest='data') +curl_parser.add_argument('-X', '--request', dest='method') +curl_parser.add_argument('-d', '--data', '--data-raw', dest='data') curl_parser.add_argument('-u', '--user', dest='auth') @@ -66,7 +66,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): if not parsed_url.scheme: url = 'http://' + url - result = {'method': parsed_args.method.upper(), 'url': url} + method = parsed_args.method or 'GET' + + result = {'method': method.upper(), 'url': url} headers = [] cookies = {} @@ -90,5 +92,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): result['cookies'] = cookies if parsed_args.data: result['body'] = parsed_args.data + if not parsed_args.method: + # if the "data" is specified but the "method" is not specified, + # the default method is 'POST' + result['method'] = 'POST' return result diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 50e1bfd5f..299a51efe 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -141,6 +141,31 @@ class CurlToRequestKwargsTest(unittest.TestCase): } self._test_command(curl_command, expected_result) + def test_post_data_raw(self): + curl_command = ( + "curl 'https://www.example.org/' --data-raw 'excerptLength=200&ena" + "bleDidYouMean=true&sortCriteria=ffirstz32xnamez32x201740686%20asc" + "ending&queryFunctions=%5B%5D&rankingFunctions=%5B%5D'" + ) + expected_result = { + "method": "POST", + "url": "https://www.example.org/", + "body": ( + "excerptLength=200&enableDidYouMean=true&sortCriteria=ffirstz3" + "2xnamez32x201740686%20ascending&queryFunctions=%5B%5D&ranking" + "Functions=%5B%5D") + } + self._test_command(curl_command, expected_result) + + def test_explicit_get_with_data(self): + curl_command = 'curl httpbin.org/anything -X GET --data asdf' + expected_result = { + "method": "GET", + "url": "http://httpbin.org/anything", + "body": "asdf" + } + self._test_command(curl_command, expected_result) + def test_patch(self): curl_command = ( 'curl "https://example.com/api/fake" -u "username:password" -H "Ac' From 23906b6bee953d9bc5dd8042e785711b11840797 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Mon, 29 Jun 2020 18:21:05 +0530 Subject: [PATCH 133/568] refactor: move TypedDict types to types.py - rename LOGGER -> logger - remove self._write_to_transport from Stream class and handle all transport related activities inside HTTP2ClientProtocol class --- scrapy/core/http2/protocol.py | 69 +++++++----- scrapy/core/http2/stream.py | 206 +++++++++++++++++----------------- scrapy/core/http2/types.py | 30 +++++ setup.py | 2 +- 4 files changed, 174 insertions(+), 133 deletions(-) create mode 100644 scrapy/core/http2/types.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 7fb935f10..0b3e5d304 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,6 +2,7 @@ import ipaddress import itertools import logging from collections import deque +from typing import Union, Dict from h2.config import H2Configuration from h2.connection import H2Connection @@ -9,20 +10,18 @@ from h2.events import ( DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) +from h2.exceptions import ProtocolError from twisted.internet.protocol import connectionDone, Protocol from twisted.internet.ssl import Certificate from scrapy.core.http2.stream import Stream, StreamCloseReason +from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request -LOGGER = logging.getLogger(__name__) +logger = logging.getLogger(__name__) class H2ClientProtocol(Protocol): - # TODO: - # 1. Check for user-agent while testing - # 2. Handle case when received events have StreamID = 0 (applied to H2Connection) - def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) @@ -37,7 +36,7 @@ class H2ClientProtocol(Protocol): self.stream_id_count = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs - self.streams = {} + self.streams: Dict[int, Stream] = {} # Boolean to keep track the connection is made # If requests are received before connection is made @@ -46,9 +45,11 @@ class H2ClientProtocol(Protocol): self.is_connection_made = False self._pending_request_stream_pool = deque() - # Some meta data of this connection - # initialized when connection is successfully made - self._metadata = { + # Save an instance of ProtocolError raised by hyper-h2 + # We pass this instance to the streams ResponseFailed() failure + self._protocol_error: Union[None, ProtocolError] = None + + self._metadata: H2ConnectionMetadataDict = { 'certificate': None, 'ip_address': None } @@ -68,7 +69,6 @@ class H2ClientProtocol(Protocol): request=request, connection=self.conn, conn_metadata=self._metadata, - write_to_transport=self._write_to_transport, cb_close=self._stream_close_cb ) @@ -89,10 +89,10 @@ class H2ClientProtocol(Protocol): data = self.conn.data_to_send() self.transport.write(data) - LOGGER.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address'])) + logger.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address'])) - def request(self, _request: Request): - stream = self._new_stream(_request) + def request(self, request: Request): + stream = self._new_stream(request) d = stream.get_response() # If connection is not yet established then add the @@ -109,7 +109,7 @@ class H2ClientProtocol(Protocol): sending some data now: we should open with the connection preamble. """ self.destination = self.transport.getPeer() - LOGGER.info('Connection made to {}'.format(self.destination)) + logger.info('Connection made to {}'.format(self.destination)) self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) @@ -119,21 +119,37 @@ class H2ClientProtocol(Protocol): self.is_connection_made = True def dataReceived(self, data): - events = self.conn.receive_data(data) - self._handle_events(events) - self._write_to_transport() + try: + events = self.conn.receive_data(data) + self._handle_events(events) + except ProtocolError as e: + # TODO: In case of InvalidBodyLengthError -- terminate only one stream + + # Save this error as ultimately the connection will be dropped + # internally by hyper-h2. Saved error will be passed to all the streams + # closed with the connection. + self._protocol_error = e + + # We lose the transport connection here + self.transport.loseConnection() + finally: + self._write_to_transport() def connectionLost(self, reason=connectionDone): """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ # Pop all streams which were pending and were not yet started - for stream_id in list(self.streams): - self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST, reason) + # NOTE: Stream.close() pops the element from the streams dictionary + # which raises `RuntimeError: dictionary changed size during iteration` + # Hence, we copy the streams into a list. + for stream in list(self.streams.values()): + stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error) self.conn.close_connection() - LOGGER.warning("Connection lost with reason " + str(reason)) + if not reason.check(connectionDone): + logger.warning("Connection lost with reason " + str(reason)) def _handle_events(self, events): """Private method which acts as a bridge between the events @@ -144,7 +160,7 @@ class H2ClientProtocol(Protocol): triggered by sending data """ for event in events: - LOGGER.debug(event) + logger.debug(event) if isinstance(event, DataReceived): self.data_received(event) elif isinstance(event, ResponseReceived): @@ -158,16 +174,14 @@ class H2ClientProtocol(Protocol): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) else: - LOGGER.info("Received unhandled event {}".format(event)) + logger.info("Received unhandled event {}".format(event)) # Event handler functions starts here def data_received(self, event: DataReceived): - stream_id = event.stream_id - self.streams[stream_id].receive_data(event.data, event.flow_controlled_length) + self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) def response_received(self, event: ResponseReceived): - stream_id = event.stream_id - self.streams[stream_id].receive_headers(event.headers) + self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged): # Send off all the pending requests @@ -175,8 +189,7 @@ class H2ClientProtocol(Protocol): self._send_pending_requests() def stream_ended(self, event: StreamEnded): - stream_id = event.stream_id - self.streams[stream_id].close(StreamCloseReason.ENDED) + self.streams[event.stream_id].close(StreamCloseReason.ENDED) def stream_reset(self, event: StreamReset): self.streams[event.stream_id].close(StreamCloseReason.RESET) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a26a33918..da0181d52 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,7 +1,7 @@ import logging -from enum import IntFlag, auto +from enum import Enum from io import BytesIO -from typing import Dict +from typing import Callable, List from urllib.parse import urlparse from h2.connection import H2Connection @@ -10,45 +10,31 @@ from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred, CancelledError from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -# for python < 3.8 -- typing.TypedDict is undefined -from typing_extensions import TypedDict +from scrapy.core.http2.types import H2ConnectionMetadataDict, H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes - -class _ResponseTypedDict(TypedDict): - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. - body: BytesIO - - # The amount of data received that counts against the flow control - # window - flow_controlled_size: int - - # Headers received after sending the request - headers: Headers +logger = logging.getLogger(__name__) -LOGGER = logging.getLogger(__name__) - - -class StreamCloseReason(IntFlag): +class StreamCloseReason(Enum): # Received a StreamEnded event - ENDED = auto() + ENDED = 1 # Received a StreamReset event -- ended abruptly - RESET = auto() + RESET = 2 # Transport connection was lost - CONNECTION_LOST = auto() + CONNECTION_LOST = 3 # Expected response body size is more than allowed limit - MAXSIZE_EXCEEDED = auto() + MAXSIZE_EXCEEDED = 4 - # When the response deferred is cancelled - CANCELLED = auto() + # When the response deferred is cancelled by the client + # (happens when client called response_deferred.cancel()) + CANCELLED = 5 class Stream: @@ -63,34 +49,30 @@ class Stream: """ def __init__( - self, - stream_id: int, - request: Request, - connection: H2Connection, - conn_metadata: Dict, - write_to_transport, - cb_close, - download_maxsize=0, - download_warnsize=0, - fail_on_data_loss=True + self, + stream_id: int, + request: Request, + connection: H2Connection, + conn_metadata: H2ConnectionMetadataDict, + cb_close: Callable[[int], None], + download_maxsize: int = 0, + download_warnsize: int = 0, + fail_on_data_loss: bool = True ): """ Arguments: - stream_id {int} -- For one HTTP/2 connection each stream is + stream_id -- For one HTTP/2 connection each stream is uniquely identified by a single integer - request {Request} -- HTTP request - connection {H2Connection} -- HTTP/2 connection this stream belongs to. - conn_metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection - write_to_transport {callable} -- Method used to write & send data to the server - This method should be used whenever some frame is to be sent to the server. - cb_close {callable} -- Method called when this stream is closed + request -- HTTP request + connection -- HTTP/2 connection this stream belongs to. + conn_metadata -- Reference to dictionary having metadata of HTTP/2 connection + cb_close -- Method called when this stream is closed to notify the TCP connection instance. """ self.stream_id = stream_id self._request = request self._conn = connection self._conn_metadata = conn_metadata - self._write_to_transport = write_to_transport self._cb_close = cb_close self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) @@ -119,7 +101,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: _ResponseTypedDict = { + self._response: H2ResponseDict = { 'body': BytesIO(), 'flow_controlled_size': 0, 'headers': Headers({}) @@ -136,6 +118,25 @@ class Stream: __repr__ = __str__ + @property + def _log_warnsize(self) -> bool: + """Checks if we have received data which exceeds the download warnsize + and whether we have not already logged about it. + + Returns: + True if both the above conditions hold true + False if any of the conditions is false + """ + content_length_header = int(self._response['headers'].get(b'Content-Length', -1)) + return ( + self._download_warnsize + and ( + self._response['flow_controlled_size'] > self._download_warnsize + or content_length_header > self._download_warnsize + ) + and not self._reached_warnsize + ) + def get_response(self): """Simply return a Deferred which fires when response from the asynchronous request is available @@ -166,10 +167,7 @@ class Stream: def initiate_request(self): headers = self._get_request_headers() self._conn.send_headers(self.stream_id, headers, end_stream=False) - self._write_to_transport() - self.request_sent = True - self.send_data() def send_data(self): @@ -211,20 +209,19 @@ class Stream: self.remaining_content_length = self.remaining_content_length - chunk_size self.remaining_content_length = max(0, self.remaining_content_length) - LOGGER.debug("{} sending {}/{} data bytes ({} frames) to {}".format( - self, - self.content_length - self.remaining_content_length, self.content_length, - data_frames_sent, - self._conn_metadata['ip_address']) + logger.debug( + "{stream} sending {received}/{expected} data bytes ({frames} frames) to {ip_address}".format( + stream=self, + received=self.content_length - self.remaining_content_length, + expected=self.content_length, + frames=data_frames_sent, + ip_address=self._conn_metadata['ip_address']) ) # End the stream if no more data needs to be send if self.remaining_content_length == 0: self._conn.end_stream(self.stream_id) - # Write data to transport -- Empty the outstanding data - self._write_to_transport() - # Q. What about the rest of the data? # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame @@ -240,24 +237,21 @@ class Stream: self._response['body'].write(data) self._response['flow_controlled_size'] += flow_controlled_length + # We check maxsize here in case the Content-Length header was not received if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize: - # Clear buffer earlier to avoid keeping data in memory for a long time - self._response['body'].truncate(0) self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) return - if self._download_warnsize \ - and self._response['flow_controlled_size'] > self._download_warnsize \ - and not self._reached_warnsize: + if self._log_warnsize: self._reached_warnsize = True - warning_msg = ('Received more ({bytes}) bytes than download ', - 'warn size ({warnsize}) in request {request}') + warning_msg = 'Received more ({bytes}) bytes than download ' \ + + 'warn size ({warnsize}) in request {request}' warning_args = { 'bytes': self._response['flow_controlled_size'], 'warnsize': self._download_warnsize, 'request': self._request } - LOGGER.warning(warning_msg, warning_args) + logger.warning(warning_msg.format(**warning_args)) # Acknowledge the data received self._conn.acknowledge_received_data( @@ -275,23 +269,27 @@ class Stream: self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) return - if self._download_warnsize and expected_size > self._download_warnsize: - warning_msg = ("Expected response size ({size}) larger than ", - "download warn size ({warnsize}) in request {request}.") + if self._log_warnsize: + self._reached_warnsize = True + warning_msg = 'Expected response size ({size}) larger than ' \ + + 'download warn size ({warnsize}) in request {request}' warning_args = { - 'size': expected_size, 'warnsize': self._download_warnsize, + 'size': expected_size, + 'warnsize': self._download_warnsize, 'request': self._request } - LOGGER.warning(warning_msg, warning_args) + logger.warning(warning_msg.format(**warning_args)) def reset_stream(self, reason=StreamCloseReason.RESET): """Close this stream by sending a RST_FRAME to the remote peer""" if self.stream_closed_local: raise StreamClosedError(self.stream_id) + # Clear buffer earlier to avoid keeping data in memory for a long time + self._response['body'].truncate(0) + self.stream_closed_local = True self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) - self._write_to_transport() self.close(reason) def _is_data_lost(self) -> bool: @@ -302,8 +300,11 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason, failure=None): + def close(self, reason: StreamCloseReason, error: Exception = None): """Based on the reason sent we will handle each case. + + Arguments: + reason -- One if StreamCloseReason """ if self.stream_closed_server: raise StreamClosedError(self.stream_id) @@ -311,35 +312,16 @@ class Stream: self._cb_close(self.stream_id) self.stream_closed_server = True - # Do nothing if the response deferred was cancelled flags = None if b'Content-Length' not in self._response['headers']: - # Missing Content-Length - PotentialDataLoss + # Missing Content-Length - {twisted.web.http.PotentialDataLoss} flags = ['partial'] - elif self._is_data_lost(): - if self._fail_on_dataloss: - self._deferred_response.errback(ResponseFailed([Failure()])) - return - else: - flags = ['dataloss'] - if reason is StreamCloseReason.ENDED: - self._fire_response_deferred(flags) - - # Stream was abruptly ended here - elif reason is StreamCloseReason.CANCELLED: - # Client has cancelled the request. Remove all the data - # received and fire the response deferred with no flags set - self._response['body'].truncate(0) - self._response['headers'].clear() - self._fire_response_deferred() - - elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST): - if failure is None: - self._deferred_response.errback(ResponseFailed([Failure()])) - else: - self._deferred_response.errback(failure) - elif reason is StreamCloseReason.MAXSIZE_EXCEEDED: + # NOTE: Order of handling the events is important here + # As we immediately cancel the request when maxsize is exceeded while + # receiving DATA_FRAME's when we have received the headers (not + # having Content-Length) + if reason is StreamCloseReason.MAXSIZE_EXCEEDED: expected_size = int(self._response['headers'].get(b'Content-Length', -1)) error_msg = ("Cancelling download of {url}: expected response " "size ({size}) larger than download max size ({maxsize}).") @@ -349,14 +331,34 @@ class Stream: 'maxsize': self._download_maxsize } - LOGGER.error(error_msg, error_args) + logger.error(error_msg, error_args) self._deferred_response.errback(CancelledError(error_msg.format(**error_args))) - def _fire_response_deferred(self, flags=None): + elif reason is StreamCloseReason.ENDED: + self._fire_response_deferred(flags) + + # Stream was abruptly ended here + elif reason is StreamCloseReason.CANCELLED: + # Client has cancelled the request. Remove all the data + # received and fire the response deferred with no flags set + + # NOTE: The data is already flushed in Stream.reset_stream() called + # immediately when the stream needs to be cancelled + + # There maybe no :status in headers, we make + # HTTP Status Code: 499 - Client Closed Request + self._response['headers'][':status'] = '499' + self._fire_response_deferred() + + elif reason in (StreamCloseReason.RESET, StreamCloseReason.CONNECTION_LOST): + self._deferred_response.errback(ResponseFailed([ + error if error else Failure() + ])) + + def _fire_response_deferred(self, flags: List[str] = None): """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" - # TODO: Update Client Side Status Codes here body = self._response['body'].getvalue() response_cls = responsetypes.from_args( @@ -365,11 +367,7 @@ class Stream: body=body ) - # If there is no :status in headers - # (happens when client called response_deferred.cancel()) - # HTTP Status Code: 499 - Client Closed Request - status = self._response['headers'].get(':status', '499') - + status = self._response['headers'][':status'] response = response_cls( url=self._request.url, status=status, diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py new file mode 100644 index 000000000..f28bf9472 --- /dev/null +++ b/scrapy/core/http2/types.py @@ -0,0 +1,30 @@ +from io import BytesIO +from ipaddress import IPv4Address, IPv6Address +from typing import Union + +from twisted.internet.ssl import Certificate +# for python < 3.8 -- typing.TypedDict is undefined +from typing_extensions import TypedDict + +from scrapy.http.headers import Headers + + +class H2ConnectionMetadataDict(TypedDict): + """Some meta data of this connection + initialized when connection is successfully made + """ + certificate: Union[None, Certificate] + ip_address: Union[None, IPv4Address, IPv6Address] + + +class H2ResponseDict(TypedDict): + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. + body: BytesIO + + # The amount of data received that counts against the flow control + # window + flow_controlled_size: int + + # Headers received after sending the request + headers: Headers diff --git a/setup.py b/setup.py index 575c74e7f..8e50733e6 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ setup( ], python_requires='>=3.5.2', install_requires=[ - 'Twisted[http2]>=17.9.0' + 'Twisted[http2]>=17.9.0', 'cryptography>=2.0', 'cssselect>=0.9.1', 'lxml>=3.5.0', From 90a7007f8818dbc224dbe51b95f07199cb730204 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Mon, 29 Jun 2020 18:29:31 +0530 Subject: [PATCH 134/568] test: warnsize logs, no content header, dataloss --- tests/test_http2_client_protocol.py | 167 +++++++++++++++++++++++----- 1 file changed, 142 insertions(+), 25 deletions(-) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0f3730c9e..0a2719d23 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,26 +1,26 @@ -# TODO: Add test cases for -# 1. No Content Length response header -# 2. Cancel Response Deferred import json import os import random +import re import shutil import string +from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks, DeferredList +from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint from twisted.internet.protocol import Factory from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate +from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.http import Request as TxRequest -from twisted.web.resource import Resource -from twisted.web.server import Site +from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol from scrapy.http import Request, Response, JsonRequest -from tests.mockserver import ssl_context_factory +from scrapy.utils.python import to_bytes, to_unicode +from tests.mockserver import ssl_context_factory, LeafResource def generate_random_string(size): @@ -35,7 +35,7 @@ def make_html_body(val): <h1>Hello from HTTP2<h1> <p>{}</p> </html>'''.format(val) - return bytes(response, 'utf-8') + return to_bytes(response) class Data: @@ -54,9 +54,8 @@ class Data: JSON_SMALL = {'data': STR_SMALL} JSON_LARGE = {'data': STR_LARGE} - -class LeafResource(Resource): - isLeaf = True + DATALOSS = b'Dataloss Content' + NO_CONTENT_LENGTH = b'This response do not have any content-length header' class GetDataHtmlSmall(LeafResource): @@ -80,9 +79,9 @@ class PostDataJsonMixin: 'extra-data': extra_data } for k, v in request.requestHeaders.getAllRawHeaders(): - response['request-headers'][k.decode('utf-8')] = v[0].decode('utf-8') + response['request-headers'][to_unicode(k)] = to_unicode(v[0]) - response_bytes = bytes(json.dumps(response), 'utf-8') + response_bytes = to_bytes(json.dumps(response)) request.setHeader('Content-Type', 'application/json') return response_bytes @@ -97,6 +96,31 @@ class PostDataJsonLarge(LeafResource, PostDataJsonMixin): return self.make_response(request, Data.EXTRA_LARGE) +class Dataloss(LeafResource): + + def render_GET(self, request: TxRequest): + request.setHeader(b"Content-Length", b"1024") + self.deferRequest(request, 0, self._delayed_render, request) + return NOT_DONE_YET + + @staticmethod + def _delayed_render(request: TxRequest): + request.write(Data.DATALOSS) + request.finish() + + +class NoContentLengthHeader(LeafResource): + def render_GET(self, request: TxRequest): + request.requestHeaders.removeHeader('Content-Length') + self.deferRequest(request, 0, self._delayed_render, request) + return NOT_DONE_YET + + @staticmethod + def _delayed_render(request: TxRequest): + request.write(Data.NO_CONTENT_LENGTH) + request.finish() + + def get_client_certificate(key_file, certificate_file): with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -118,6 +142,9 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'post-data-json-small', PostDataJsonSmall()) r.putChild(b'post-data-json-large', PostDataJsonLarge()) + + r.putChild(b'dataloss', Dataloss()) + r.putChild(b'no-content-length-header', NoContentLengthHeader()) return r @inlineCallbacks @@ -172,10 +199,10 @@ class Https2ClientProtocolTestCase(TestCase): return DeferredList(d_list, fireOnOneErrback=True) def _check_GET( - self, - request: Request, - expected_body, - expected_status + self, + request: Request, + expected_body, + expected_status ): def check_response(response: Response): self.assertEqual(response.status, expected_status) @@ -220,11 +247,11 @@ class Https2ClientProtocolTestCase(TestCase): ) def _check_POST_json( - self, - request: Request, - expected_request_body, - expected_extra_data, - expected_status: int + self, + request: Request, + expected_request_body, + expected_extra_data, + expected_status: int ): d = self.client.request(request) @@ -237,7 +264,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(len(response.body), content_length) # Parse the body - body = json.loads(response.body.decode('utf-8')) + body = json.loads(to_unicode(response.body)) self.assertIn('request-body', body) self.assertIn('extra-data', body) self.assertIn('request-headers', body) @@ -251,11 +278,12 @@ class Https2ClientProtocolTestCase(TestCase): # Check if headers were sent successfully request_headers = body['request-headers'] for k, v in request.headers.items(): - k_str = k.decode('utf-8') + k_str = to_unicode(k) self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], v[0].decode('utf-8')) + self.assertEqual(request_headers[k_str], to_unicode(v[0])) d.addCallback(assert_response) + d.addErrback(self.fail) return d def test_POST_small_json(self): @@ -310,6 +338,95 @@ class Https2ClientProtocolTestCase(TestCase): d = self.client.request(request) d.addCallback(assert_response) + d.addErrback(self.fail) d.cancel() return d + + def test_download_maxsize_exceeded(self): + request = Request(url=self.get_url('/get-data-html-large'), meta={'download_maxsize': 1000}) + + def assert_cancelled_error(failure): + self.assertIsInstance(failure.value, CancelledError) + + d = self.client.request(request) + d.addCallback(self.fail) + d.addErrback(assert_cancelled_error) + return d + + # TODO: Test in multiple requests if one request fails due to dataloss + # remaining request do not fail (change expected behaviour) + # Can be done only when hyper-h2 don't terminate connection over + # InvalidBodyLengthError check + def test_received_dataloss_response(self): + """In case when value of Header Content-Length != len(Received Data) + ProtocolError is raised""" + request = Request(url=self.get_url('/dataloss')) + + def assert_failure(failure: Failure): + self.assertTrue(len(failure.value.reasons) > 0) + self.assertTrue(any( + isinstance(error, InvalidBodyLengthError) + for error in failure.value.reasons + )) + + d = self.client.request(request) + d.addCallback(self.fail) + d.addErrback(assert_failure) + return d + + def test_missing_content_length_header(self): + request = Request(url=self.get_url('/no-content-length-header')) + + def assert_content_length(response: Response): + self.assertEqual(response.status, 200) + self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + self.assertIn('partial', response.flags) + self.assertNotIn('Content-Length', response.headers) + + d = self.client.request(request) + d.addCallback(assert_content_length) + d.addErrback(self.fail) + return d + + @inlineCallbacks + def _check_log_warnsize( + self, + request, + warn_pattern, + expected_body + ): + with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm: + response = yield self.client.request(request) + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, expected_body) + + # Check the warning is raised only once for this request + self.assertEqual(sum( + len(re.findall(warn_pattern, log)) + for log in cm.output + ), 1) + + @inlineCallbacks + def test_log_expected_warnsize(self): + request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000}) + warn_pattern = re.compile( + r'Expected response size \(\d*\) larger than ' + r'download warn size \(1000\) in request {}'.format(request) + ) + + yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) + + @inlineCallbacks + def test_log_received_warnsize(self): + request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10}) + warn_pattern = re.compile( + r'Received more \(\d*\) bytes than download ' + r'warn size \(10\) in request {}'.format(request) + ) + + yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) From e46b47c365b27e3f3f383a102122361db0a86a42 Mon Sep 17 00:00:00 2001 From: Aditya Kumar <k.aditya00@gmail.com> Date: Mon, 29 Jun 2020 18:35:13 +0530 Subject: [PATCH 135/568] Renew the localhost certificate for tests (#4650) Validity Not Before: Jun 28 12:54:15 2020 GMT Not After : Jun 28 12:54:15 2021 GMT Subject: C = IE, O = Scrapy, CN = localhost --- tests/keys/localhost.crt | 36 ++++++++++++++-------------- tests/keys/localhost.key | 52 ++++++++++++++++++++-------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt index 13c5b5bd6..0cf5256d8 100644 --- a/tests/keys/localhost.crt +++ b/tests/keys/localhost.crt @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV -BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x -NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD -VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K -o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y -njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY -ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/ -8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52 -gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O -BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j -xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES -9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i -jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+ -TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own -tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM -am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U -Mjybr/hQK3mnVdE= +MIIDRTCCAi2gAwIBAgIUGoISfeW3LwSWHC52ORXdZY9pNLswDQYJKoZIhvcNAQEL +BQAwMjELMAkGA1UEBhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9j +YWxob3N0MB4XDTIwMDYyODEyNTQxNVoXDTIxMDYyODEyNTQxNVowMjELMAkGA1UE +BhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvCLxfTEQuIdf8JhiHrbVkGHYrNSK +2XD2TCPaSIpJ2KKlFUrIz3A9tWlOfLnWabS5od89yOebhYj4DN/Qm2TViGg1mtWe +pD1K2YWd1Af+hhAw5D+TpW2RH9TVhX7Ey5osWcl+0uy+RlKZE8qum72xi1vxWOmH +wYw06iN8klQ3JfP2/eLRXBQjsh7WW0dbJ7yLvG6UFz1RbhFTtlxeIMenzNsHaMg7 +56Ru57/MMbaBwdBttXVzJDQ7imo8njuxDMszliC/QgIdBUBFzA2LB5qpr+v+laDN +cN9t9Q9stsu446dFnRoofxJjMFW7lLu6h/lwP5r0kfeUkMDhXJ4mb6KwfwIDAQAB +o1MwUTAdBgNVHQ4EFgQUVEdXn8ha2FA73zcy1Ia0FQMzMEYwHwYDVR0jBBgwFoAU +VEdXn8ha2FA73zcy1Ia0FQMzMEYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B +AQsFAAOCAQEAZpGBPsexMD+IwcMNIgc7FiaJsb8E30C9vWxgdnkpapi9zLJ4yiHQ +VxkV9RTezUEADkaDj+2qFveamWTzJLnphgaaUpVeMcYACPhRVOYXidNrZyTmHIsX +FwaTzAggW6CP7JxAcpxH0f9+NWFCZI36FihRdwuWyvrUl7rsXaexu0SOI/Ck0oWf +2IW+jo67TSmcbte+J8wq77DX32mVLb/2nqpItH4T2Di+XjVBARACVOSdgdlo7lZE +W8mSEXqP2BVx8JGG8X1znNLHcmjVj4EtkpH0wkYzpC4cvGkTsUcU7CU7ZyVUp+Bb +dPMVxyRKWfAjRJc8o5Ot1mgHrx5coOtzAA== -----END CERTIFICATE----- diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key index da975e6d3..8fc373bdd 100644 --- a/tests/keys/localhost.key +++ b/tests/keys/localhost.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5 -ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4 -JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz -vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2 -RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY -b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/ -XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0 -gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk -i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu -Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss -ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4 -hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD -7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa -EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf -uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG -HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR -UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO -j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri -fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ -6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI -xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g -DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f -npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM -8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6 -J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj -Z+pMCFLbwAbFZ9Qb/NFlUv0= +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8IvF9MRC4h1/w +mGIettWQYdis1IrZcPZMI9pIiknYoqUVSsjPcD21aU58udZptLmh3z3I55uFiPgM +39CbZNWIaDWa1Z6kPUrZhZ3UB/6GEDDkP5OlbZEf1NWFfsTLmixZyX7S7L5GUpkT +yq6bvbGLW/FY6YfBjDTqI3ySVDcl8/b94tFcFCOyHtZbR1snvIu8bpQXPVFuEVO2 +XF4gx6fM2wdoyDvnpG7nv8wxtoHB0G21dXMkNDuKajyeO7EMyzOWIL9CAh0FQEXM +DYsHmqmv6/6VoM1w3231D2y2y7jjp0WdGih/EmMwVbuUu7qH+XA/mvSR95SQwOFc +niZvorB/AgMBAAECggEAHVpSVRb/pdqxNEeCH4qlHWa2uJhcpXpDYzPAzcqNpPgT +S5QkaoD3j8NDVKBl/I4O3FuJNzwzfo0VLmUJFgWQbzzbCDJGExfhArkfG8K3ilEi +X6ovrgK/PrklKzPRHncKbmPKnrwDH9OpQHZB8diRx81rhVTCModehh1NRUNQa2I1 +QzFC7uyXx3duoIsI5QXVeEGuwHZfqIY/z+9SscdVFL6elXTPFUzBzcmAqQgdgWKN +HXgX22LE0rAu8NnRvOZZWt4/nOjvlCFCPTB11NgthmKlVnsx4H7gpQ2OPh4bZ+0W +birVEtZ3E1jxoGvw1FzxyqqpGkcanRMa8QWzK4JwuQKBgQDrgclpkqZrgHB/TC1p +hLvsdflGI2SGs+c/mYR3GEjf0kJtI88WL5fj1QezdkDyOpwxFvnLslswfzdtzvis +vksGysV35vhMPQUcmWhvzA7Pdxdv4BZr+ckER0SAYBBxg9KYZyxewGb5XzB8Cz2o +8V+YpwrMAOYGuXHTfafv4CKlTQKBgQDMgetvV9/E3HNtKsATiPIwT3e1MzyPXigq +12NkHSZa6s4yqm/h/fSUn54sJbhx+OtRRhktOo0aB34tcogtrJyClvCPdRAP/4Qi +M43FjKo2cWiubWvtWlOZU04bpClG324q420rK7dCA2stID/Fa0sMQgAAyPH8TGMo +gbvyrk4W+wKBgQDMIOnYZTF0epaH8BponJFaqwMOhTzr+OGW4dTMebMotZG4EdK8 +kzIfW5XaOsSecKjTb+vCYGzkA1CjEEPBTwuu7nDstblAM5/Lozi/tmqb7sjUwrIM +kyxmVfONJjb6fV07lioCUtiui5B15DRkzBqlMRyNqLW43GJKA19d7rN4/QKBgCzy +kRBTu/bEjQn9T2H7w18i2CiXLkREaYeg91NVpMxutwsjspt0+YCA5H7He5ZxIycl +xPrP15tU8kKC3bNMMMny6sRc8j7R5fSuaAZ3OCHnIx7TJdlw9NbKHGyu0/Ojv87l +VWUbopd7sN6mK930CvaSuvVxNN5C27hXazuXW8ppAoGBANcWsenNKpCJgF0cNPHX +abPaWfcs5FKMNz8gEdGk3B1z/KBpYz59smPwurYVCXaWE6iv99sDOP7CVneF02sV +SqyNzVhcVSG788uB3CwnpEvm7ydoH89L5dvYekAHP8RJulhWCK45lXkHLiYGKvhv +PWuPk5VX+qF78JhUhPO3nfnu -----END PRIVATE KEY----- From 26ab3e4137ddee3c643ae63c0709529efc698433 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Tue, 30 Jun 2020 06:44:20 +0530 Subject: [PATCH 136/568] feat: FIFO policy to handle large no. of requests - add required test -- test by sending 1000 requests - increase test timeout to 180 seconds to account for tests taking long time --- scrapy/core/http2/protocol.py | 87 +++++++++++++++++++---------- scrapy/core/http2/stream.py | 26 ++++++++- tests/test_http2_client_protocol.py | 51 ++++++++++++++++- 3 files changed, 128 insertions(+), 36 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 0b3e5d304..4de80c05e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -38,13 +38,16 @@ class H2ClientProtocol(Protocol): # Streams are stored in a dictionary keyed off their stream IDs self.streams: Dict[int, Stream] = {} - # Boolean to keep track the connection is made - # If requests are received before connection is made - # we keep all requests in a pool and send them as the connection - # is made - self.is_connection_made = False + # If requests are received before connection is made we keep + # all requests in a pool and send them as the connection is made self._pending_request_stream_pool = deque() + # Counter to keep track of opened stream. This counter + # is used to make sure that not more than MAX_CONCURRENT_STREAMS + # streams are opened which leads to ProtocolError + # We use simple FIFO policy to handle pending requests + self._active_streams = 0 + # Save an instance of ProtocolError raised by hyper-h2 # We pass this instance to the streams ResponseFailed() failure self._protocol_error: Union[None, ProtocolError] = None @@ -54,10 +57,46 @@ class H2ClientProtocol(Protocol): 'ip_address': None } + @property + def is_connected(self): + """Boolean to keep track of the connection status. + This is used while initiating pending streams to make sure + that we initiate stream only during active HTTP/2 Connection + """ + return bool(self.transport.connected) + + @property + def allowed_max_concurrent_streams(self) -> int: + """We keep total two streams for client (sending data) and + server side (receiving data) for a single request. To be safe + we choose the minimum. Since this value can change in event + RemoteSettingsChanged we make variable a property. + """ + return min( + self.conn.local_settings.max_concurrent_streams, + self.conn.remote_settings.max_concurrent_streams + ) + + def _send_pending_requests(self): + """Initiate all pending requests from the deque following FIFO + We make sure that at any time {allowed_max_concurrent_streams} + streams are active. + """ + while ( + self._pending_request_stream_pool + and self._active_streams < self.allowed_max_concurrent_streams + and self.is_connected + ): + self._active_streams += 1 + stream = self._pending_request_stream_pool.popleft() + stream.initiate_request() + def _stream_close_cb(self, stream_id: int): """Called when stream is closed completely """ - self.streams.pop(stream_id, None) + self.streams.pop(stream_id) + self._active_streams -= 1 + self._send_pending_requests() def _new_stream(self, request: Request): """Instantiates a new Stream object @@ -75,13 +114,6 @@ class H2ClientProtocol(Protocol): self.streams[stream.stream_id] = stream return stream - def _send_pending_requests(self): - # TODO: handle MAX_CONCURRENT_STREAMS - # Initiate all pending requests - while self._pending_request_stream_pool: - stream = self._pending_request_stream_pool.popleft() - stream.initiate_request() - def _write_to_transport(self): """ Write data to the underlying transport connection from the HTTP2 connection instance if any @@ -89,19 +121,12 @@ class H2ClientProtocol(Protocol): data = self.conn.data_to_send() self.transport.write(data) - logger.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address'])) - def request(self, request: Request): stream = self._new_stream(request) d = stream.get_response() - # If connection is not yet established then add the - # stream to pool or initiate request - if self.is_connection_made: - stream.initiate_request() - else: - self._pending_request_stream_pool.append(stream) - + # Add the stream to the request pool + self._pending_request_stream_pool.append(stream) return d def connectionMade(self): @@ -116,7 +141,6 @@ class H2ClientProtocol(Protocol): self.conn.initiate_connection() self._write_to_transport() - self.is_connection_made = True def dataReceived(self, data): try: @@ -144,7 +168,10 @@ class H2ClientProtocol(Protocol): # which raises `RuntimeError: dictionary changed size during iteration` # Hence, we copy the streams into a list. for stream in list(self.streams.values()): - stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error) + if stream.request_sent: + stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error) + else: + stream.close(StreamCloseReason.INACTIVE) self.conn.close_connection() @@ -160,7 +187,6 @@ class H2ClientProtocol(Protocol): triggered by sending data """ for event in events: - logger.debug(event) if isinstance(event, DataReceived): self.data_received(event) elif isinstance(event, ResponseReceived): @@ -174,7 +200,7 @@ class H2ClientProtocol(Protocol): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) else: - logger.info("Received unhandled event {}".format(event)) + logger.debug("Received unhandled event {}".format(event)) # Event handler functions starts here def data_received(self, event: DataReceived): @@ -184,8 +210,8 @@ class H2ClientProtocol(Protocol): self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged): - # Send off all the pending requests - # as now we have established a proper HTTP/2 connection + # Send off all the pending requests as now we have + # established a proper HTTP/2 connection self._send_pending_requests() def stream_ended(self, event: StreamEnded): @@ -195,9 +221,8 @@ class H2ClientProtocol(Protocol): self.streams[event.stream_id].close(StreamCloseReason.RESET) def window_updated(self, event: WindowUpdated): - stream_id = event.stream_id - if stream_id != 0: - self.streams[stream_id].receive_window_update() + if event.stream_id != 0: + self.streams[event.stream_id].receive_window_update() else: # Send leftover data for all the streams for stream in self.streams.values(): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index da0181d52..19b1825e4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -8,6 +8,7 @@ from h2.connection import H2Connection from h2.errors import ErrorCodes from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred, CancelledError +from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed @@ -19,6 +20,15 @@ from scrapy.responsetypes import responsetypes logger = logging.getLogger(__name__) +class InactiveStreamClosed(ConnectionClosed): + """Connection was closed without sending request headers + of the stream. This happens when a stream is waiting for other + streams to close and connection is lost.""" + + def __init__(self, request: Request): + self.request = request + + class StreamCloseReason(Enum): # Received a StreamEnded event ENDED = 1 @@ -32,10 +42,13 @@ class StreamCloseReason(Enum): # Expected response body size is more than allowed limit MAXSIZE_EXCEEDED = 4 - # When the response deferred is cancelled by the client + # Response deferred is cancelled by the client # (happens when client called response_deferred.cancel()) CANCELLED = 5 + # Connection lost and the stream was not initiated + INACTIVE = 6 + class Stream: """Represents a single HTTP/2 Stream. @@ -108,8 +121,12 @@ class Stream: } def _cancel(_): - # Close this stream as gracefully as possible :) - self.reset_stream(StreamCloseReason.CANCELLED) + # Close this stream as gracefully as possible + # Check if the stream has started + if self.request_sent: + self.reset_stream(StreamCloseReason.CANCELLED) + else: + self.close(StreamCloseReason.CANCELLED) self._deferred_response = Deferred(_cancel) @@ -355,6 +372,9 @@ class Stream: error if error else Failure() ])) + elif reason is StreamCloseReason.INACTIVE: + self._deferred_response.errback(InactiveStreamClosed(self._request)) + def _fire_response_deferred(self, flags: List[str] = None): """Builds response from the self._response dict and fires the response deferred callback with the diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0a2719d23..0cb32dda6 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -18,6 +18,7 @@ from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.core.http2.stream import InactiveStreamClosed from scrapy.http import Request, Response, JsonRequest from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource @@ -174,10 +175,14 @@ class Https2ClientProtocolTestCase(TestCase): client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) + # Increase the total time taken for each tests + self.timeout = 180 # default is 120 seconds + @inlineCallbacks def tearDown(self): - yield self.client.transport.loseConnection() - yield self.client.transport.abortConnection() + if self.client.is_connected: + yield self.client.transport.loseConnection() + yield self.client.transport.abortConnection() yield self.server.stopListening() shutil.rmtree(self.temp_directory) @@ -430,3 +435,45 @@ class Https2ClientProtocolTestCase(TestCase): ) yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) + + def test_max_concurrent_streams(self): + """Send 1000 requests to check if we can handle + very large number of request + """ + + def get_deferred(): + return self._check_GET( + Request(self.get_url('/get-data-html-small')), + Data.HTML_SMALL, + 200 + ) + + return self._check_repeat(get_deferred, 1000) + + def test_inactive_stream(self): + """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS + by default is 100. After sending the first 100 requests we close the + connection.""" + d_list = [] + + def assert_inactive_stream(failure): + self.assertIsNotNone(failure.check(InactiveStreamClosed)) + + # Send 100 request (we do not check the result) + for _ in range(100): + d = self.client.request(Request(self.get_url('/get-data-html-small'))) + d.addBoth(lambda _: None) + d_list.append(d) + + # Now send 10 extra request and save the response deferred in a list + for _ in range(10): + d = self.client.request(Request(self.get_url('/get-data-html-small'))) + d.addCallback(self.fail) + d.addErrback(assert_inactive_stream) + d_list.append(d) + + # Close the connection now to fire all the extra 10 requests errback + # with InactiveStreamClosed + self.client.transport.abortConnection() + + return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) From 50dd9271b4566785430106cfa9384d51103f73d9 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Tue, 30 Jun 2020 07:17:48 +0530 Subject: [PATCH 137/568] fix: disable redundant logs - while testing the job exceeded the maximum log length and was terminated - reduce the number of requests from 20 to 10 --- scrapy/core/http2/stream.py | 8 -------- tests/test_http2_client_protocol.py | 27 ++++++++++++--------------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 19b1825e4..8d0c6d94d 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -226,14 +226,6 @@ class Stream: self.remaining_content_length = self.remaining_content_length - chunk_size self.remaining_content_length = max(0, self.remaining_content_length) - logger.debug( - "{stream} sending {received}/{expected} data bytes ({frames} frames) to {ip_address}".format( - stream=self, - received=self.content_length - self.remaining_content_length, - expected=self.content_length, - frames=data_frames_sent, - ip_address=self._conn_metadata['ip_address']) - ) # End the stream if no more data needs to be send if self.remaining_content_length == 0: diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0cb32dda6..79c129d11 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -175,9 +175,6 @@ class Https2ClientProtocolTestCase(TestCase): client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) - # Increase the total time taken for each tests - self.timeout = 180 # default is 120 seconds - @inlineCallbacks def tearDown(self): if self.client.is_connected: @@ -231,21 +228,21 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(self.get_url('/get-data-html-large')) return self._check_GET(request, Data.HTML_LARGE, 200) - def _check_GET_x20(self, *args, **kwargs): + def _check_GET_x10(self, *args, **kwargs): def get_deferred(): return self._check_GET(*args, **kwargs) - return self._check_repeat(get_deferred, 20) + return self._check_repeat(get_deferred, 10) - def test_GET_small_body_x20(self): - return self._check_GET_x20( + def test_GET_small_body_x10(self): + return self._check_GET_x10( Request(self.get_url('/get-data-html-small')), Data.HTML_SMALL, 200 ) - def test_GET_large_body_x20(self): - return self._check_GET_x20( + def test_GET_large_body_x10(self): + return self._check_GET_x10( Request(self.get_url('/get-data-html-large')), Data.HTML_LARGE, 200 @@ -309,24 +306,24 @@ class Https2ClientProtocolTestCase(TestCase): 200 ) - def _check_POST_json_x20(self, *args, **kwargs): + def _check_POST_json_x10(self, *args, **kwargs): def get_deferred(): return self._check_POST_json(*args, **kwargs) - return self._check_repeat(get_deferred, 20) + return self._check_repeat(get_deferred, 10) - def test_POST_small_json_x20(self): + def test_POST_small_json_x10(self): request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) - return self._check_POST_json_x20( + return self._check_POST_json_x10( request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200 ) - def test_POST_large_json_x20(self): + def test_POST_large_json_x10(self): request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) - return self._check_POST_json_x20( + return self._check_POST_json_x10( request, Data.JSON_LARGE, Data.EXTRA_LARGE, From 5b88c522ac1b1a9ba1588573d90cf3bc01339282 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Tue, 30 Jun 2020 12:18:21 -0300 Subject: [PATCH 138/568] Simplify dataclass example in item loader docs --- docs/topics/loaders.rst | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index e921395d2..9c82bb4d9 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -88,29 +88,17 @@ item loaders: unless a pre-populated item is passed to the loader, fields will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods. -Given the way that item loaders store data internally, one approach -to overcome this is to define items using the :func:`~dataclasses.field` -function, with ``list`` as the ``default_factory`` argument:: +One approach to overcome this is to define items using the +:func:`~dataclasses.field` function, with a ``default`` argument:: from dataclasses import dataclass, field + from typing import Optional @dataclass class InventoryItem: - name: str = field(default_factory=list) - price: float = field(default_factory=list) - stock: int = field(default_factory=list) - -Note that in order to keep the example simple, the types do not match -completely. A more accurate but verbose definition would be:: - - from dataclasses import dataclass, field - from typing import List, Union - - @dataclass - class InventoryItem: - name: Union[str, List[str]] = field(default_factory=list) - price: Union[float, List[float]] = field(default_factory=list) - stock: Union[int, List[int]] = field(default_factory=list) + name: Optional[str] = field(default=None) + price: Optional[float] = field(default=None) + stock: Optional[int] = field(default=None) .. _topics-loaders-processors: From 7b1ad995a4996babf9a019815dd7256a1cbfa044 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Wed, 1 Jul 2020 10:45:36 +0530 Subject: [PATCH 139/568] test: query params, certificate & ip_address - refactor from str.format() to f-strings --- scrapy/core/http2/protocol.py | 12 ++-- scrapy/core/http2/stream.py | 46 +++++--------- scrapy/core/http2/types.py | 6 +- setup.py | 7 ++- tests/test_http2_client_protocol.py | 96 +++++++++++++++++++++++------ 5 files changed, 108 insertions(+), 59 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 4de80c05e..3438c99f0 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -122,6 +122,9 @@ class H2ClientProtocol(Protocol): self.transport.write(data) def request(self, request: Request): + if not isinstance(request, Request): + raise TypeError(f'Expected type scrapy.http.Request but received {request.__class__.__name__}') + stream = self._new_stream(request) d = stream.get_response() @@ -134,9 +137,7 @@ class H2ClientProtocol(Protocol): sending some data now: we should open with the connection preamble. """ self.destination = self.transport.getPeer() - logger.info('Connection made to {}'.format(self.destination)) - - self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) + logger.info(f'Connection made to {self.destination}') self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) self.conn.initiate_connection() @@ -200,7 +201,7 @@ class H2ClientProtocol(Protocol): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) else: - logger.debug("Received unhandled event {}".format(event)) + logger.debug(f'Received unhandled event {event}') # Event handler functions starts here def data_received(self, event: DataReceived): @@ -214,6 +215,9 @@ class H2ClientProtocol(Protocol): # established a proper HTTP/2 connection self._send_pending_requests() + # Update certificate when our HTTP/2 connection is established + self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) + def stream_ended(self, event: StreamEnded): self.streams[event.stream_id].close(StreamCloseReason.ENDED) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8d0c6d94d..f4a90a753 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -131,7 +131,7 @@ class Stream: self._deferred_response = Deferred(_cancel) def __str__(self): - return "Stream(id={})".format(repr(self.stream_id)) + return f'Stream(id={self.stream_id})' __repr__ = __str__ @@ -167,13 +167,15 @@ class Stream: url = urlparse(self._request.url) # Make sure pseudo-headers comes before all the other headers + path = url.path + if url.query: + path += '?' + url.query + headers = [ (':method', self._request.method), (':authority', url.netloc), - - # TODO: Check if scheme can be 'http' for HTTP/2 ? (':scheme', 'https'), - (':path', url.path), + (':path', path), ] for name, value in self._request.headers.items(): @@ -253,14 +255,9 @@ class Stream: if self._log_warnsize: self._reached_warnsize = True - warning_msg = 'Received more ({bytes}) bytes than download ' \ - + 'warn size ({warnsize}) in request {request}' - warning_args = { - 'bytes': self._response['flow_controlled_size'], - 'warnsize': self._download_warnsize, - 'request': self._request - } - logger.warning(warning_msg.format(**warning_args)) + warning_msg = f"Received more ({self._response['flow_controlled_size']}) bytes than download " \ + + f'warn size ({self._download_warnsize}) in request {self._request}' + logger.warning(warning_msg) # Acknowledge the data received self._conn.acknowledge_received_data( @@ -280,14 +277,9 @@ class Stream: if self._log_warnsize: self._reached_warnsize = True - warning_msg = 'Expected response size ({size}) larger than ' \ - + 'download warn size ({warnsize}) in request {request}' - warning_args = { - 'size': expected_size, - 'warnsize': self._download_warnsize, - 'request': self._request - } - logger.warning(warning_msg.format(**warning_args)) + warning_msg = f'Expected response size ({expected_size}) larger than ' \ + + f'download warn size ({self._download_warnsize}) in request {self._request}' + logger.warning(warning_msg) def reset_stream(self, reason=StreamCloseReason.RESET): """Close this stream by sending a RST_FRAME to the remote peer""" @@ -332,16 +324,10 @@ class Stream: # having Content-Length) if reason is StreamCloseReason.MAXSIZE_EXCEEDED: expected_size = int(self._response['headers'].get(b'Content-Length', -1)) - error_msg = ("Cancelling download of {url}: expected response " - "size ({size}) larger than download max size ({maxsize}).") - error_args = { - 'url': self._request.url, - 'size': expected_size, - 'maxsize': self._download_maxsize - } - - logger.error(error_msg, error_args) - self._deferred_response.errback(CancelledError(error_msg.format(**error_args))) + error_msg = f'Cancelling download of {self._request.url}: expected response ' \ + f'size ({expected_size}) larger than download max size ({self._download_maxsize}).' + logger.error(error_msg) + self._deferred_response.errback(CancelledError(error_msg)) elif reason is StreamCloseReason.ENDED: self._fire_response_deferred(flags) diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index f28bf9472..c0961cd3a 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -1,6 +1,6 @@ from io import BytesIO from ipaddress import IPv4Address, IPv6Address -from typing import Union +from typing import Union, Optional from twisted.internet.ssl import Certificate # for python < 3.8 -- typing.TypedDict is undefined @@ -13,8 +13,8 @@ class H2ConnectionMetadataDict(TypedDict): """Some meta data of this connection initialized when connection is successfully made """ - certificate: Union[None, Certificate] - ip_address: Union[None, IPv4Address, IPv6Address] + certificate: Optional[Certificate] + ip_address: Optional[Union[IPv4Address, IPv6Address]] class H2ResponseDict(TypedDict): diff --git a/setup.py b/setup.py index 8e50733e6..47c5906e4 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from os.path import dirname, join - from pkg_resources import parse_version from setuptools import setup, find_packages, __version__ as setuptools_version + with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: version = f.read().decode('ascii').strip() @@ -25,11 +25,12 @@ if has_environment_marker_platform_impl_support(): 'PyPyDispatcher>=2.1.0', ] + setup( name='Scrapy', version=version, url='https://scrapy.org', - project_urls={ + project_urls = { 'Documentation': 'https://docs.scrapy.org/', 'Source': 'https://github.com/scrapy/scrapy', 'Tracker': 'https://github.com/scrapy/scrapy/issues', @@ -83,4 +84,4 @@ setup( 'typing_extensions>=3.7' ], extras_require=extras_require, -) +) \ No newline at end of file diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 79c129d11..c6a9bd5fb 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -4,13 +4,15 @@ import random import re import shutil import string +from ipaddress import IPv4Address +from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint from twisted.internet.protocol import Factory -from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate +from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.http import Request as TxRequest @@ -21,7 +23,7 @@ from scrapy.core.http2.protocol import H2ClientProtocol from scrapy.core.http2.stream import InactiveStreamClosed from scrapy.http import Request, Response, JsonRequest from scrapy.utils.python import to_bytes, to_unicode -from tests.mockserver import ssl_context_factory, LeafResource +from tests.mockserver import ssl_context_factory, LeafResource, Status def generate_random_string(size): @@ -32,10 +34,10 @@ def generate_random_string(size): def make_html_body(val): - response = '''<html> + response = f'''<html> <h1>Hello from HTTP2<h1> -<p>{}</p> -</html>'''.format(val) +<p>{val}</p> +</html>''' return to_bytes(response) @@ -122,6 +124,17 @@ class NoContentLengthHeader(LeafResource): request.finish() +class QueryParams(LeafResource): + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'application/json') + + query_params = {} + for k, v in request.args.items(): + query_params[to_unicode(k)] = to_unicode(v[0]) + + return to_bytes(json.dumps(query_params)) + + def get_client_certificate(key_file, certificate_file): with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -146,6 +159,8 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'dataloss', Dataloss()) r.putChild(b'no-content-length-header', NoContentLengthHeader()) + r.putChild(b'status', Status()) + r.putChild(b'query-params', QueryParams()) return r @inlineCallbacks @@ -189,7 +204,7 @@ class Https2ClientProtocolTestCase(TestCase): :return: Complete url """ assert len(path) > 0 and (path[0] == '/' or path[0] == '&') - return "{}://{}:{}{}".format(self.scheme, self.hostname, self.port_number, path) + return f'{self.scheme}://{self.hostname}:{self.port_number}{path}' @staticmethod def _check_repeat(get_deferred, count): @@ -210,7 +225,6 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, expected_status) self.assertEqual(response.body, expected_body) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) content_length = int(response.headers.get('Content-Length')) self.assertEqual(len(response.body), content_length) @@ -260,7 +274,6 @@ class Https2ClientProtocolTestCase(TestCase): def assert_response(response: Response): self.assertEqual(response.status, expected_status) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) content_length = int(response.headers.get('Content-Length')) self.assertEqual(len(response.body), content_length) @@ -336,7 +349,6 @@ class Https2ClientProtocolTestCase(TestCase): def assert_response(response: Response): self.assertEqual(response.status, 499) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) d = self.client.request(request) d.addCallback(assert_response) @@ -356,10 +368,6 @@ class Https2ClientProtocolTestCase(TestCase): d.addErrback(assert_cancelled_error) return d - # TODO: Test in multiple requests if one request fails due to dataloss - # remaining request do not fail (change expected behaviour) - # Can be done only when hyper-h2 don't terminate connection over - # InvalidBodyLengthError check def test_received_dataloss_response(self): """In case when value of Header Content-Length != len(Received Data) ProtocolError is raised""" @@ -384,7 +392,6 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, 200) self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) self.assertIn('partial', response.flags) self.assertNotIn('Content-Length', response.headers) @@ -404,7 +411,6 @@ class Https2ClientProtocolTestCase(TestCase): response = yield self.client.request(request) self.assertEqual(response.status, 200) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) self.assertEqual(response.body, expected_body) # Check the warning is raised only once for this request @@ -417,8 +423,8 @@ class Https2ClientProtocolTestCase(TestCase): def test_log_expected_warnsize(self): request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000}) warn_pattern = re.compile( - r'Expected response size \(\d*\) larger than ' - r'download warn size \(1000\) in request {}'.format(request) + rf'Expected response size \(\d*\) larger than ' + rf'download warn size \(1000\) in request {request}' ) yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) @@ -427,8 +433,8 @@ class Https2ClientProtocolTestCase(TestCase): def test_log_received_warnsize(self): request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10}) warn_pattern = re.compile( - r'Received more \(\d*\) bytes than download ' - r'warn size \(10\) in request {}'.format(request) + rf'Received more \(\d*\) bytes than download ' + rf'warn size \(10\) in request {request}' ) yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) @@ -474,3 +480,55 @@ class Https2ClientProtocolTestCase(TestCase): self.client.transport.abortConnection() return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) + + def test_invalid_request_type(self): + with self.assertRaises(TypeError): + self.client.request('https://InvalidDataTypePassed.com') + + def test_query_parameters(self): + params = { + 'a': generate_random_string(20), + 'b': generate_random_string(20), + 'c': generate_random_string(20), + 'd': generate_random_string(20) + } + request = Request(self.get_url(f'/query-params?{urlencode(params)}')) + + def assert_query_params(response: Response): + data = json.loads(to_unicode(response.body)) + self.assertEqual(data, params) + + d = self.client.request(request) + d.addCallback(assert_query_params) + d.addErrback(self.fail) + + return d + + def test_status_codes(self): + def assert_response_status(response: Response, expected_status: int): + self.assertEqual(response.status, expected_status) + + d_list = [] + for status in [200, 404]: + request = Request(self.get_url(f'/status?n={status}')) + d = self.client.request(request) + d.addCallback(assert_response_status, status) + d.addErrback(self.fail) + d_list.append(d) + + return DeferredList(d_list, fireOnOneErrback=True) + + def test_response_has_correct_certificate_ip_address(self): + request = Request(self.get_url('/status?n=200')) + + def assert_metadata(response: Response): + self.assertEqual(response.request, request) + self.assertIsInstance(response.certificate, Certificate) + self.assertIsInstance(response.ip_address, IPv4Address) + self.assertEqual(str(response.ip_address), '127.0.0.1') + + d = self.client.request(request) + d.addCallback(assert_metadata) + d.addErrback(self.fail) + + return d From 7fc80671a84144ad36c3d5332aa822ef3be00781 Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Wed, 1 Jul 2020 13:32:17 +0530 Subject: [PATCH 140/568] Update schemaless URI support --- scrapy/core/downloader/handlers/http11.py | 12 ++++++++---- scrapy/core/downloader/webclient.py | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 73e56c87d..15de8cdbd 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -7,7 +7,7 @@ import warnings from contextlib import suppress from io import BytesIO from time import time -from urllib.parse import urldefrag, urlparse +from urllib.parse import urldefrag, urlunparse from twisted.internet import defer, protocol, ssl from twisted.internet.endpoints import TCP4ClientEndpoint @@ -255,7 +255,7 @@ class ScrapyProxyAgent(Agent): bindAddress=bindAddress, pool=pool, ) - self._proxyURI = URI.fromBytes(urlparse(proxyURI)._replace(scheme=b'http').geturl()) + self._proxyURI = URI.fromBytes(proxyURI) def request(self, method, uri, headers=None, bodyProducer=None): """ @@ -297,7 +297,7 @@ class ScrapyAgent: bindaddress = request.meta.get('bindaddress') or self._bindAddress proxy = request.meta.get('proxy') if proxy: - _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) + proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams @@ -319,9 +319,13 @@ class ScrapyAgent: pool=self._pool, ) else: + proxyScheme = b'http' if not proxyScheme else proxyScheme + proxyHost = to_bytes(proxyHost, encoding='ascii') + proxyPort = to_bytes(str(proxyPort), encoding='ascii') + proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) return self._ProxyAgent( reactor=reactor, - proxyURI=to_bytes(proxy, encoding='ascii'), + proxyURI=to_bytes(proxyURI, encoding='ascii'), connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool, diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 355045d74..af49e78ce 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,5 +1,6 @@ from time import time from urllib.parse import urlparse, urlunparse, urldefrag +from re import match from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient @@ -32,6 +33,8 @@ def _parse(url): and is ascii-only. """ url = url.strip() + if not match(r'^\w+://', url): + url = '//' + url parsed = urlparse(url) return _parsed_url_args(parsed) From 006a945214422da22f645b4416263124377adc5e Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Wed, 1 Jul 2020 13:32:58 +0530 Subject: [PATCH 141/568] Update schemaless http proxy test --- tests/test_downloader_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 5854659dd..9441be736 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -741,7 +741,7 @@ class Http11ProxyTestCase(HttpProxyTestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.body, b'http://example.com') - http_proxy = self.getURL('').replace('http:', '') + http_proxy = self.getURL('').replace('http://', '') request = Request('http://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) From 7b1d3c35ea3bfde2ac7fc69a2a26bbcb94aec1bf Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Wed, 1 Jul 2020 11:54:39 +0300 Subject: [PATCH 142/568] Minor updates --- docs/topics/feed-exports.rst | 4 ++-- scrapy/extensions/feedexport.py | 34 +++++++++++++++++---------------- scrapy/utils/conf.py | 4 ++++ tests/test_feedexport.py | 23 ++++++---------------- tests/test_utils_conf.py | 4 ++++ 5 files changed, 34 insertions(+), 35 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 3da56821e..0b659f30e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -442,7 +442,7 @@ If assigned an integer number higher than ``0``, Scrapy generates multiple outpu storing up to the specified number of items in each output file. When generating multiple output files, you must use at least one of the following -placeholders in :setting:`FEED_URI` to indicate how the different output file names are +placeholders in the feed URI to indicate how the different output file names are generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created @@ -457,7 +457,7 @@ For instance, if your settings include:: And your :command:`crawl` command line is:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json The command line above can generate a directory tree like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 2312c994e..5908987a3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -242,7 +242,6 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -253,7 +252,7 @@ class FeedExporter: def open_spider(self, spider): for uri, feed in self.feeds.items(): - uri_params = self._get_uri_params(spider, feed['uri_params'], None) + uri_params = self._get_uri_params(spider, feed['uri_params']) self.slots.append(self._start_new_batch( batch_id=1, uri=uri % uri_params, @@ -299,7 +298,7 @@ class FeedExporter: def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): """ Redirect the output data stream to a new file. - Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified. + Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed @@ -331,14 +330,15 @@ class FeedExporter: def item_scraped(self, item, spider): slots = [] - for idx, slot in enumerate(self.slots): + for slot in self.slots: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one - if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \ - and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count', - self.storage_batch_item_count): + if ( + self.feeds[slot.uri_template]['batch_item_count'] + and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] + ): uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( @@ -369,15 +369,17 @@ class FeedExporter: def _settings_are_valid(self, uri): """ - If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s - to distinguish different files of partial output + If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain + %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ - if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: - return True - logger.error( - '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified' - ) - return False + for uri_template, values in self.feeds.items(): + if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']): + logger.error( + '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting ' + 'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template) + ) + return False + return True def _storage_supported(self, uri): scheme = urlparse(uri).scheme @@ -404,7 +406,7 @@ class FeedExporter: def _get_storage(self, uri): return self._get_instance(self.storages[urlparse(uri).scheme], uri) - def _get_uri_params(self, spider, uri_params, slot): + def _get_uri_params(self, spider, uri_params, slot=None): params = {} for k in dir(spider): params[k] = getattr(spider, k) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 5921f82bf..0e02f0f28 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -115,6 +115,10 @@ def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) + out.setdefault( + "batch_item_count", + out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')) + ) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) if settings["FEED_EXPORT_INDENT"] is None: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 542cce70f..db14b20b9 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): - """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """ + """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1329,7 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase): ], 'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'), 'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')], - 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], } settings = { @@ -1352,13 +1351,6 @@ class BatchDeliveriesTest(FeedExportTestBase): 'fields': ['foo', 'bar'], 'encoding': 'utf-8', }, - os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { - 'format': 'jsonlines', - 'indent': None, - 'fields': ['foo', 'bar'], - 'encoding': 'utf-8', - 'batch_item_count': 0, - }, }, 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } @@ -1369,19 +1361,16 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): - items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] - + items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})] formats = { - 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'), - '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], + 'json': ['[{"foo": "FOO"}]'.encode('utf-8'), + '[{"foo": "FOO1"}]'.encode('utf-8')], } - settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { - 'format': 'jsonlines', + os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + 'format': 'json', 'indent': None, - 'fields': ['foo', 'bar'], 'encoding': 'utf-8', 'batch_item_count': 1, }, diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index e5d3ef582..95ec2b64a 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -149,6 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, "FEED_URI_PARAMS": (1, 2, 3, 4), + "FEED_STORAGE_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -157,6 +158,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "indent": 42, "store_empty": True, "uri_params": (1, 2, 3, 4), + "batch_item_count": 2, }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -169,6 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, + "FEED_STORAGE_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -177,6 +180,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "indent": 42, "store_empty": True, "uri_params": None, + "batch_item_count": 2, }) From 065b9b1170fe249fbfce51c87631e5572127df21 Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Wed, 1 Jul 2020 15:53:29 +0530 Subject: [PATCH 143/568] Update regex import --- scrapy/core/downloader/webclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index af49e78ce..8b6f87c3f 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,6 +1,6 @@ +import re from time import time from urllib.parse import urlparse, urlunparse, urldefrag -from re import match from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient @@ -33,7 +33,7 @@ def _parse(url): and is ascii-only. """ url = url.strip() - if not match(r'^\w+://', url): + if not re.match(r'^\w+://', url): url = '//' + url parsed = urlparse(url) return _parsed_url_args(parsed) From c361fe0d3b80ff8a9f88adc05e730d5e469db225 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Wed, 1 Jul 2020 18:14:44 +0530 Subject: [PATCH 144/568] feat: check for invalid hostname - Initiating requests having hostname or (ip_address, port) different from the peer to which HTTP/2 connection is made can lead to closing the whole connection and close out all the pending streams. - This change aims to fix that problem - Add required tests - Save hostname & port in H2ConnectionMetadataDict --- scrapy/core/http2/protocol.py | 26 ++++++------ scrapy/core/http2/stream.py | 50 +++++++++++++++++++--- scrapy/core/http2/types.py | 11 +++++ tests/test_http2_client_protocol.py | 64 +++++++++++++++++++++++------ 4 files changed, 118 insertions(+), 33 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 3438c99f0..5de516482 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,7 +2,7 @@ import ipaddress import itertools import logging from collections import deque -from typing import Union, Dict +from typing import Dict, Optional from h2.config import H2Configuration from h2.connection import H2Connection @@ -26,10 +26,6 @@ class H2ClientProtocol(Protocol): config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) - # Address of the server we are connected to - # these are updated when connection is successfully made - self.destination = None - # ID of the next request stream # Following the convention made by hyper-h2 each client ID # will be odd. @@ -50,11 +46,13 @@ class H2ClientProtocol(Protocol): # Save an instance of ProtocolError raised by hyper-h2 # We pass this instance to the streams ResponseFailed() failure - self._protocol_error: Union[None, ProtocolError] = None + self._protocol_error: Optional[ProtocolError] = None self._metadata: H2ConnectionMetadataDict = { 'certificate': None, - 'ip_address': None + 'ip_address': None, + 'hostname': None, + 'port': None } @property @@ -123,7 +121,7 @@ class H2ClientProtocol(Protocol): def request(self, request: Request): if not isinstance(request, Request): - raise TypeError(f'Expected type scrapy.http.Request but received {request.__class__.__name__}') + raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__name__}') stream = self._new_stream(request) d = stream.get_response() @@ -136,9 +134,11 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ - self.destination = self.transport.getPeer() - logger.info(f'Connection made to {self.destination}') - self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) + destination = self.transport.getPeer() + logger.debug('Connection made to {}'.format(destination)) + self._metadata['ip_address'] = ipaddress.ip_address(destination.host) + self._metadata['port'] = destination.port + self._metadata['hostname'] = self.transport.transport.addr[0] self.conn.initiate_connection() self._write_to_transport() @@ -148,8 +148,6 @@ class H2ClientProtocol(Protocol): events = self.conn.receive_data(data) self._handle_events(events) except ProtocolError as e: - # TODO: In case of InvalidBodyLengthError -- terminate only one stream - # Save this error as ultimately the connection will be dropped # internally by hyper-h2. Saved error will be passed to all the streams # closed with the connection. @@ -201,7 +199,7 @@ class H2ClientProtocol(Protocol): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) else: - logger.debug(f'Received unhandled event {event}') + logger.debug('Received unhandled event {}'.format(event)) # Event handler functions starts here def data_received(self, event: DataReceived): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index f4a90a753..cc751b682 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -29,6 +29,17 @@ class InactiveStreamClosed(ConnectionClosed): self.request = request +class InvalidHostname(Exception): + + def __init__(self, request: Request, expected_hostname, expected_netloc): + self.request = request + self.expected_hostname = expected_hostname + self.expected_netloc = expected_netloc + + def __str__(self): + return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}' + + class StreamCloseReason(Enum): # Received a StreamEnded event ENDED = 1 @@ -49,6 +60,10 @@ class StreamCloseReason(Enum): # Connection lost and the stream was not initiated INACTIVE = 6 + # The hostname of the request is not same as of connected peer hostname + # As a result sending this request will the end the connection + INVALID_HOSTNAME = 7 + class Stream: """Represents a single HTTP/2 Stream. @@ -163,6 +178,15 @@ class Stream: """ return self._deferred_response + def check_request_url(self) -> bool: + # Make sure that we are sending the request to the correct URL + url = urlparse(self._request.url) + return ( + url.netloc == self._conn_metadata['hostname'] + or url.netloc == f'{self._conn_metadata["hostname"]}:{self._conn_metadata["port"]}' + or url.netloc == f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}' + ) + def _get_request_headers(self): url = urlparse(self._request.url) @@ -184,10 +208,15 @@ class Stream: return headers def initiate_request(self): - headers = self._get_request_headers() - self._conn.send_headers(self.stream_id, headers, end_stream=False) - self.request_sent = True - self.send_data() + if self.check_request_url(): + headers = self._get_request_headers() + self._conn.send_headers(self.stream_id, headers, end_stream=False) + self.request_sent = True + self.send_data() + else: + # Close this stream calling the response errback + # Note that we have not sent any headers + self.close(StreamCloseReason.INVALID_HOSTNAME) def send_data(self): """Called immediately after the headers are sent. Here we send all the @@ -310,6 +339,9 @@ class Stream: if self.stream_closed_server: raise StreamClosedError(self.stream_id) + if not isinstance(reason, StreamCloseReason): + raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__name__}') + self._cb_close(self.stream_id) self.stream_closed_server = True @@ -353,6 +385,13 @@ class Stream: elif reason is StreamCloseReason.INACTIVE: self._deferred_response.errback(InactiveStreamClosed(self._request)) + elif reason is StreamCloseReason.INVALID_HOSTNAME: + self._deferred_response.errback(InvalidHostname( + self._request, + self._conn_metadata['hostname'], + f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}' + )) + def _fire_response_deferred(self, flags: List[str] = None): """Builds response from the self._response dict and fires the response deferred callback with the @@ -365,10 +404,9 @@ class Stream: body=body ) - status = self._response['headers'][':status'] response = response_cls( url=self._request.url, - status=status, + status=self._response['headers'][':status'], headers=self._response['headers'], body=body, request=self._request, diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index c0961cd3a..dd7b1187b 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -14,8 +14,19 @@ class H2ConnectionMetadataDict(TypedDict): initialized when connection is successfully made """ certificate: Optional[Certificate] + + # Address of the server we are connected to which + # is updated when HTTP/2 connection is made successfully ip_address: Optional[Union[IPv4Address, IPv6Address]] + # Name of the peer HTTP/2 connection is established + hostname: Optional[str] + + port: Optional[int] + + # Both ip_address and hostname are used by the Stream before + # initiating the request to verify that the base address + class H2ResponseDict(TypedDict): # Data received frame by frame from the server is appended diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index c6a9bd5fb..8f9fbe6fd 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -10,7 +10,7 @@ from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError -from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint +from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.protocol import Factory from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure @@ -20,7 +20,7 @@ from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.core.http2.stream import InactiveStreamClosed +from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -135,7 +135,7 @@ class QueryParams(LeafResource): return to_bytes(json.dumps(query_params)) -def get_client_certificate(key_file, certificate_file): +def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -171,19 +171,16 @@ class Https2ClientProtocolTestCase(TestCase): # Start server for testing self.hostname = u'localhost' - if self.scheme == 'https': - context_factory = ssl_context_factory(self.key_file, self.certificate_file) - server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) - else: - server_endpoint = TCP4ServerEndpoint(reactor, 0, interface=self.hostname) + context_factory = ssl_context_factory(self.key_file, self.certificate_file) + server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) self.server = yield server_endpoint.listen(self.site) self.port_number = self.server.getHost().port # Connect H2 client with server - client_certificate = get_client_certificate(self.key_file, self.certificate_file) + self.client_certificate = get_client_certificate(self.key_file, self.certificate_file) client_options = optionsForClientTLS( hostname=self.hostname, - trustRoot=client_certificate, + trustRoot=self.client_certificate, acceptableProtocols=[b'h2'] ) h2_client_factory = Factory.forProtocol(H2ClientProtocol) @@ -440,8 +437,8 @@ class Https2ClientProtocolTestCase(TestCase): yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) def test_max_concurrent_streams(self): - """Send 1000 requests to check if we can handle - very large number of request + """Send 500 requests at one to check if we can handle + very large number of request. """ def get_deferred(): @@ -451,7 +448,7 @@ class Https2ClientProtocolTestCase(TestCase): 200 ) - return self._check_repeat(get_deferred, 1000) + return self._check_repeat(get_deferred, 500) def test_inactive_stream(self): """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS @@ -524,6 +521,10 @@ class Https2ClientProtocolTestCase(TestCase): def assert_metadata(response: Response): self.assertEqual(response.request, request) self.assertIsInstance(response.certificate, Certificate) + self.assertIsNotNone(response.certificate.original) + self.assertEqual(response.certificate.getIssuer(), self.client_certificate.getIssuer()) + self.assertTrue(response.certificate.getPublicKey().matches(self.client_certificate.getPublicKey())) + self.assertIsInstance(response.ip_address, IPv4Address) self.assertEqual(str(response.ip_address), '127.0.0.1') @@ -532,3 +533,40 @@ class Https2ClientProtocolTestCase(TestCase): d.addErrback(self.fail) return d + + def _check_invalid_netloc(self, url): + request = Request(url) + + def assert_invalid_hostname(failure: Failure): + self.assertIsNotNone(failure.check(InvalidHostname)) + error_msg = str(failure.value) + self.assertIn('localhost', error_msg) + self.assertIn('127.0.0.1', error_msg) + self.assertIn(str(request), error_msg) + + d = self.client.request(request) + d.addCallback(self.fail) + d.addErrback(assert_invalid_hostname) + return d + + def test_invalid_hostname(self): + return self._check_invalid_netloc('https://notlocalhost.notlocalhostdomain') + + def test_invalid_host_port(self): + port = self.port_number + 1 + return self._check_invalid_netloc(f'https://127.0.0.1:{port}') + + def test_connection_stays_with_invalid_requests(self): + d_list = [ + self.test_invalid_hostname(), + self.test_invalid_host_port(), + self._check_GET(Request(self.get_url('/get-data-html-small')), Data.HTML_SMALL, 200), + self._check_POST_json( + JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL), + Data.JSON_SMALL, + Data.EXTRA_SMALL, + 200 + ) + ] + + return DeferredList(d_list, fireOnOneErrback=True) From 4acdc2e5d623c6330235647aaad817e77eaad800 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Wed, 1 Jul 2020 20:15:33 +0530 Subject: [PATCH 145/568] refactor: use __qualname__, () for large strings --- scrapy/core/http2/protocol.py | 2 +- scrapy/core/http2/stream.py | 24 +++++++++++++++--------- tests/test_http2_client_protocol.py | 9 ++------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 5de516482..a3dfdb76e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -121,7 +121,7 @@ class H2ClientProtocol(Protocol): def request(self, request: Request): if not isinstance(request, Request): - raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__name__}') + raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}') stream = self._new_stream(request) d = stream.get_response() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index cc751b682..f45ddc04e 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -146,7 +146,7 @@ class Stream: self._deferred_response = Deferred(_cancel) def __str__(self): - return f'Stream(id={self.stream_id})' + return f'Stream(id={self.stream_id!r})' __repr__ = __str__ @@ -190,11 +190,11 @@ class Stream: def _get_request_headers(self): url = urlparse(self._request.url) - # Make sure pseudo-headers comes before all the other headers path = url.path if url.query: path += '?' + url.query + # Make sure pseudo-headers comes before all the other headers headers = [ (':method', self._request.method), (':authority', url.netloc), @@ -284,8 +284,10 @@ class Stream: if self._log_warnsize: self._reached_warnsize = True - warning_msg = f"Received more ({self._response['flow_controlled_size']}) bytes than download " \ - + f'warn size ({self._download_warnsize}) in request {self._request}' + warning_msg = ( + f'Received more ({self._response["flow_controlled_size"]}) bytes than download ' + f'warn size ({self._download_warnsize}) in request {self._request}' + ) logger.warning(warning_msg) # Acknowledge the data received @@ -306,8 +308,10 @@ class Stream: if self._log_warnsize: self._reached_warnsize = True - warning_msg = f'Expected response size ({expected_size}) larger than ' \ - + f'download warn size ({self._download_warnsize}) in request {self._request}' + warning_msg = ( + f'Expected response size ({expected_size}) larger than ' + f'download warn size ({self._download_warnsize}) in request {self._request}' + ) logger.warning(warning_msg) def reset_stream(self, reason=StreamCloseReason.RESET): @@ -340,7 +344,7 @@ class Stream: raise StreamClosedError(self.stream_id) if not isinstance(reason, StreamCloseReason): - raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__name__}') + raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}') self._cb_close(self.stream_id) self.stream_closed_server = True @@ -356,8 +360,10 @@ class Stream: # having Content-Length) if reason is StreamCloseReason.MAXSIZE_EXCEEDED: expected_size = int(self._response['headers'].get(b'Content-Length', -1)) - error_msg = f'Cancelling download of {self._request.url}: expected response ' \ - f'size ({expected_size}) larger than download max size ({self._download_maxsize}).' + error_msg = ( + f'Cancelling download of {self._request.url}: expected response ' + f'size ({expected_size}) larger than download max size ({self._download_maxsize}).' + ) logger.error(error_msg) self._deferred_response.errback(CancelledError(error_msg)) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 8f9fbe6fd..ca8a629b1 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -560,13 +560,8 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [ self.test_invalid_hostname(), self.test_invalid_host_port(), - self._check_GET(Request(self.get_url('/get-data-html-small')), Data.HTML_SMALL, 200), - self._check_POST_json( - JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL), - Data.JSON_SMALL, - Data.EXTRA_SMALL, - 200 - ) + self.test_GET_small_body(), + self.test_POST_small_json() ] return DeferredList(d_list, fireOnOneErrback=True) From af55d23167f9cec22f815bc9f9884b10a9a35f5b Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin <wrar@wrar.name> Date: Wed, 1 Jul 2020 19:46:54 +0500 Subject: [PATCH 146/568] Update the OpenSSL cipher list format link OpenSSL `ciphers(1)` is now almost empty: https://www.openssl.org/docs/manmaster/man1/ciphers.html Alternative would be linking to 1.1.1 docs specifically. --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5178f272f..8cc8806a5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -469,7 +469,7 @@ necessary to access certain HTTPS websites: for example, you may need to use ``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a specific cipher that is not included in ``DEFAULT`` if a website requires it. -.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT +.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT .. setting:: DOWNLOADER_CLIENT_TLS_METHOD From 1e245046ed8ac3d9f89860501c2da95b69aaabf6 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 2 Jul 2020 12:38:08 +0300 Subject: [PATCH 147/568] Change setting name. Add leading zeroes to batch_id. Minor fixes. --- docs/topics/feed-exports.rst | 19 +++++++++-------- scrapy/extensions/feedexport.py | 24 +++++++++++++--------- scrapy/settings/default_settings.py | 2 +- scrapy/utils/conf.py | 5 +---- tests/test_feedexport.py | 32 ++++++++++++++--------------- tests/test_utils_conf.py | 4 ++-- 6 files changed, 45 insertions(+), 41 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0b659f30e..56efa80a7 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` - * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` + * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport @@ -272,7 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition. * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` -* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` +* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. setting:: FEED_EXPORT_ENCODING @@ -432,9 +432,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl -.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT +.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT -FEED_STORAGE_BATCH_ITEM_COUNT +FEED_EXPORT_BATCH_ITEM_COUNT ----------------------------- Default: ``0`` @@ -448,16 +448,19 @@ generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created (e.g. ``2020-03-28T14-45-08.237134``) -* ``%(batch_id)s`` - gets replaced by the batch sequence number of batch - (e.g. ``2`` for the second file) +* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch. +By replacing ``x`` with an integer you set the number of leading zeroes to prevent +inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples: + ``%(batch_id)01d`` for the second batch gets replaced by ``2`` + ``%(batch_id)05d`` for the third batch gets replaced by ``00003`` For instance, if your settings include:: - FEED_STORAGE_BATCH_ITEM_COUNT=100 + FEED_EXPORT_BATCH_ITEM_COUNT=100 And your :command:`crawl` command line is:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json + scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json The command line above can generate a directory tree like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 5908987a3..adb6ea2e4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst import logging import os +import re import sys import warnings from datetime import datetime @@ -25,6 +26,7 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values + logger = logging.getLogger(__name__) @@ -245,7 +247,7 @@ class FeedExporter: for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured - if not self._settings_are_valid(uri): + if not self._settings_are_valid(): raise NotConfigured if not self._exporter_supported(feed['format']): raise NotConfigured @@ -298,7 +300,7 @@ class FeedExporter: def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): """ Redirect the output data stream to a new file. - Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified + Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed @@ -334,10 +336,10 @@ class FeedExporter: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one + # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one if ( - self.feeds[slot.uri_template]['batch_item_count'] - and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] + self.feeds[slot.uri_template]['batch_item_count'] + and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] ): uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) @@ -367,16 +369,18 @@ class FeedExporter: return True logger.error("Unknown feed format: %(format)s", {'format': format}) - def _settings_are_valid(self, uri): + def _settings_are_valid(self): """ - If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain + If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ for uri_template, values in self.feeds.items(): - if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']): + if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template): logger.error( - '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting ' - 'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template) + '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' + 'or FEEDS.batch_item_count is specified and greater than 0. For more info see:' + 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' + ''.format(uri_template) ) return False return True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 810acd5a3..0016bbe1b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_ITEM_COUNT = 0 +FEED_EXPORT_BATCH_ITEM_COUNT = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 0e02f0f28..64f9c824b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -113,12 +113,9 @@ def get_sources(use_closest=True): def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() + out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) - out.setdefault( - "batch_item_count", - out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')) - ) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) if settings["FEED_EXPORT_INDENT"] is None: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index db14b20b9..d20b40e2f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1108,7 +1108,7 @@ class FeedExportTest(FeedExportTestBase): class BatchDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(batch_time)s_#%(batch_id)s_' + _file_mark = '_%(batch_time)s_#%(batch_id)02d_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -1259,18 +1259,18 @@ class BatchDeliveriesTest(FeedExportTestBase): {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} ] settings = { - 'FEED_STORAGE_BATCH_ITEM_COUNT': 2 + 'FEED_EXPORT_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): - """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """ + """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 } crawler = get_crawler(settings_dict=settings) self.assertRaises(NotConfigured, FeedExporter, crawler) @@ -1282,7 +1282,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'FEEDS': { os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 } data = yield self.exported_no_data(settings) data = dict(data) @@ -1304,7 +1304,7 @@ class BatchDeliveriesTest(FeedExportTestBase): }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_no_data(settings) data = dict(data) @@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'encoding': 'utf-8', }, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): @@ -1398,7 +1398,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) @@ -1440,7 +1440,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, }) items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1458,7 +1458,7 @@ class BatchDeliveriesTest(FeedExportTestBase): s3 = boto3.resource('s3') my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') with MockServer() as s: runner = CrawlerRunner(Settings(settings)) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 95ec2b64a..f3ef36127 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -149,7 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, "FEED_URI_PARAMS": (1, 2, 3, 4), - "FEED_STORAGE_BATCH_ITEM_COUNT": 2, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -171,7 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, - "FEED_STORAGE_BATCH_ITEM_COUNT": 2, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { From 3199048520ebe3798c14cfa7362612b51d156b55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 2 Jul 2020 20:10:08 +0200 Subject: [PATCH 148/568] Complete Azure Pipelines CI setup --- azure-pipelines.yml | 18 ++++++------------ tests/CrawlerRunner/ip_address.py | 15 ++++++++++++++- tests/mockserver.py | 5 ++--- tests/test_commands.py | 5 +++++ tests/test_crawler.py | 13 +++++++++++++ tests/test_feedexport.py | 8 +++++++- tests/test_proxy_connect.py | 3 +++ tests/test_spiderloader/__init__.py | 23 +++++++++++++++-------- tests/test_utils_asyncio.py | 7 ++++++- tox.ini | 18 +++++++++++++++--- 10 files changed, 86 insertions(+), 29 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ffc4d549b..710e42090 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,29 +1,23 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - - +variables: + TOXENV: py pool: - vmImage: 'windows-2019' + vmImage: 'windows-latest' strategy: matrix: Python35: python.version: '3.5' - TOXENV: py35 + TOXENV: windows-pinned Python36: python.version: '3.6' - TOXENV: py36 Python37: python.version: '3.7' - TOXENV: py37 - + Python38: + python.version: '3.8' steps: - task: UsePythonVersion@0 inputs: versionSpec: '$(python.version)' displayName: 'Use Python $(python.version)' - - script: | pip install -U tox twine wheel codecov tox diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 826374cd4..ea75bc3c9 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,7 +1,9 @@ from urllib.parse import urlparse from twisted.internet import reactor -from twisted.names.client import createResolver +from twisted.names import cache, hosts as hostsModule, resolve +from twisted.names.client import Resolver +from twisted.python.runtime import platform from scrapy import Spider, Request from scrapy.crawler import CrawlerRunner @@ -10,6 +12,17 @@ from scrapy.utils.log import configure_logging from tests.mockserver import MockServer, MockDNSServer +# https://stackoverflow.com/a/32784190 +def createResolver(servers=None, resolvconf=None, hosts=None): + if hosts is None: + hosts = (b'/etc/hosts' if platform.getType() == 'posix' + else r'c:\windows\hosts') + theResolver = Resolver(resolvconf, servers) + hostResolver = hostsModule.Resolver(hosts) + L = [hostResolver, cache.CacheResolver(), theResolver] + return resolve.ResolverChain(L) + + class LocalhostSpider(Spider): name = "localhost_spider" diff --git a/tests/mockserver.py b/tests/mockserver.py index df30feab6..1f40473ba 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -247,9 +247,8 @@ class MockDNSServer: def __enter__(self): self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'], stdout=PIPE, env=get_testenv()) - host, port = self.proc.stdout.readline().strip().decode('ascii').split(":") - self.host = host - self.port = int(port) + self.host = '127.0.0.1' + self.port = int(self.proc.stdout.readline().strip().decode('ascii').split(":")[1]) return self def __exit__(self, exc_type, exc_value, traceback): diff --git a/tests/test_commands.py b/tests/test_commands.py index 24a341759..ee0e4511a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -2,6 +2,7 @@ import inspect import json import optparse import os +import platform import subprocess import sys import tempfile @@ -10,6 +11,7 @@ from os.path import exists, join, abspath from shutil import rmtree, copytree from tempfile import mkdtemp from threading import Timer +from unittest import skipIf from twisted.trial import unittest @@ -319,6 +321,9 @@ class BadSpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 78704fb2c..1a4cfe813 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,6 +4,7 @@ import platform import subprocess import sys import warnings +from unittest import skipIf from pytest import raises, mark from testfixtures import LogCapture @@ -252,6 +253,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase): }) @defer.inlineCallbacks + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_crawler_process_asyncio_enabled_true(self): with LogCapture(level=logging.DEBUG) as log: if self.reactor_pytest == 'asyncio': @@ -293,11 +297,17 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -327,6 +337,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e38644214..f7b997560 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -455,9 +455,15 @@ class FeedExportTest(unittest.TestCase): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ + def path_to_url(path): + return urljoin('file:', pathname2url(str(path))) + + def printf_escape(string): + return string.replace('%', '%%') + FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { - urljoin('file:', pathname2url(str(file_path))): feed + printf_escape(path_to_url(file_path)): feed for file_path, feed in FEEDS.items() } diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index eb4ecc91d..fc5658ae7 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,5 +1,6 @@ import json import os +import platform import re import sys from subprocess import Popen, PIPE @@ -59,6 +60,8 @@ def _wrong_credentials(proxy_url): @skipIf(sys.version_info < (3, 5, 4), "requires mitmproxy < 3.0.0, which these tests do not support") +@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), + "mitmproxy does not support Windows when running Python < 3.7") class ProxyConnectTestCase(TestCase): def setUp(self): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d922c6059..4929f1e3e 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -20,13 +20,20 @@ from scrapy.crawler import CrawlerRunner module_dir = os.path.dirname(os.path.abspath(__file__)) +def _copytree(source, target): + try: + shutil.copytree(source, target) + except shutil.Error: + pass + + class SpiderLoaderTest(unittest.TestCase): def setUp(self): orig_spiders_dir = os.path.join(module_dir, 'test_spiders') self.tmpdir = tempfile.mkdtemp() self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') - shutil.copytree(orig_spiders_dir, self.spiders_dir) + _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) self.spider_loader = SpiderLoader.from_settings(settings) @@ -124,7 +131,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.tmpdir = self.mktemp() os.mkdir(self.tmpdir) self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') - shutil.copytree(orig_spiders_dir, self.spiders_dir) + _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) @@ -134,8 +141,8 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): def test_dupename_warning(self): # copy 1 spider module so as to have duplicate spider name - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py')) with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) @@ -156,10 +163,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name # This should issue 2 warning, 1 for each duplicate spider name - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py')) - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py')) with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 295323e4d..a2114bd18 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,4 +1,6 @@ -from unittest import TestCase +import platform +import sys +from unittest import skipIf, TestCase from pytest import mark @@ -12,6 +14,9 @@ class AsyncioTest(TestCase): # the result should depend only on the pytest --reactor argument self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_install_asyncio_reactor(self): # this should do nothing install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/tox.ini b/tox.ini index 27d21ade2..f729327ca 100644 --- a/tox.ini +++ b/tox.ini @@ -63,14 +63,12 @@ basepython = pypy3 commands = py.test {posargs:--durations=10 docs scrapy tests} -[testenv:pinned] -basepython = python3 +[pinned] deps = -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 itemadapter==0.1.0 - lxml==3.5.0 parsel==1.5.0 Protego==0.1.15 PyDispatcher==2.0.5 @@ -85,6 +83,20 @@ deps = botocore==1.3.23 Pillow==3.4.2 +[testenv:pinned] +basepython = python3 +deps = + {[pinned]deps} + lxml==3.5.0 + +[testenv:windows-pinned] +basepython = python3 +deps = + {[pinned]deps} + # First lxml version that includes a Windows wheel for Python 3.5, so we do + # not need to build lxml from sources in a CI Windows job: + lxml==3.8.0 + [testenv:extra-deps] deps = {[testenv]deps} From eb937742566105f3525a9f76e4ae68cc18e9fd8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Fri, 3 Jul 2020 01:41:47 +0200 Subject: [PATCH 149/568] TrackrefTestCase.test_get_oldest: protect from lack of precision --- tests/test_utils_trackref.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 16e02f919..b8e8c3130 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,7 +1,10 @@ import unittest from io import StringIO +from time import sleep, time from unittest import mock +from twisted.trial.unittest import SkipTest + from scrapy.utils import trackref @@ -55,7 +58,18 @@ Foo 1 oldest: 0s ago\n\n''') def test_get_oldest(self): o1 = Foo() # NOQA + + o1_time = time() + o2 = Bar() # NOQA + + o3_time = time() + if o3_time <= o1_time: + sleep(0.01) + o3_time = time() + if o3_time <= o1_time: + raise SkipTest('time.time is not precise enough') + o3 = Foo() # NOQA self.assertIs(trackref.get_oldest('Foo'), o1) self.assertIs(trackref.get_oldest('Bar'), o2) From 6454d456d2bcab0828aba6d81d98f7393ab7e04d Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Fri, 3 Jul 2020 08:29:54 +0300 Subject: [PATCH 150/568] Make check of placeholder less strict --- docs/topics/feed-exports.rst | 11 ++++++----- scrapy/extensions/feedexport.py | 4 ++-- tests/test_feedexport.py | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 56efa80a7..0bb5f1733 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -448,11 +448,12 @@ generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created (e.g. ``2020-03-28T14-45-08.237134``) -* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch. -By replacing ``x`` with an integer you set the number of leading zeroes to prevent -inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples: - ``%(batch_id)01d`` for the second batch gets replaced by ``2`` - ``%(batch_id)05d`` for the third batch gets replaced by ``00003`` +* ``%(batch_id)d`` - gets replaced by the sequence number of the batch. + + Use :ref:`printf-style string formatting <python:old-string-formatting>` to + alter the number format. For example, to make the batch ID a 5-digit + number by introducing leading zeroes as needed, use ``%(batch_id)05d`` + (e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``). For instance, if your settings include:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index adb6ea2e4..e15c1a09c 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -375,9 +375,9 @@ class FeedExporter: %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ for uri_template, values in self.feeds.items(): - if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template): + if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): logger.error( - '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' + '%(batch_time)s or %(batch_id) must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' 'or FEEDS.batch_item_count is specified and greater than 0. For more info see:' 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' ''.format(uri_template) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d20b40e2f..4e0b867a4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): - """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """ + """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, From a94b30342a451b94e7f358f68ce1b1adc20723f9 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Mon, 6 Jul 2020 12:49:12 +0530 Subject: [PATCH 151/568] test: reduce test data size to 1MB --- scrapy/core/http2/stream.py | 2 -- tests/test_http2_client_protocol.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index f45ddc04e..1c856ff68 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -243,7 +243,6 @@ class Stream: bytes_to_send_size = min(window_size, self.remaining_content_length) # We now need to send a number of data frames. - data_frames_sent = 0 while bytes_to_send_size > 0: chunk_size = min(bytes_to_send_size, max_frame_size) @@ -252,7 +251,6 @@ class Stream: self._conn.send_data(self.stream_id, data_chunk, end_stream=False) - data_frames_sent += 1 bytes_to_send_size = bytes_to_send_size - chunk_size self.remaining_content_length = self.remaining_content_length - chunk_size diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index ca8a629b1..98dc98a0f 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -42,8 +42,8 @@ def make_html_body(val): class Data: - SMALL_SIZE = 1024 * 10 # 10 KB - LARGE_SIZE = (1024 ** 2) * 10 # 10 MB + SMALL_SIZE = 1024 # 1 KB + LARGE_SIZE = 1024 ** 2 # 1 MB STR_SMALL = generate_random_string(SMALL_SIZE) STR_LARGE = generate_random_string(LARGE_SIZE) From 7f5bb6b34c6a5137aa12cd4ad4f3845a8c67653f Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Mon, 6 Jul 2020 13:08:14 +0530 Subject: [PATCH 152/568] chore: add h2 to setup.py, tox.ini - Change log level for hpack to ERROR --- scrapy/utils/log.py | 3 +++ setup.py | 3 ++- tox.ini | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 51d276097..4e2671478 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -52,6 +52,9 @@ DEFAULT_LOGGING = { 'twisted': { 'level': 'ERROR', }, + 'hpack': { + 'level': 'ERROR', + }, } } diff --git a/setup.py b/setup.py index 47c5906e4..d872d6472 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,8 @@ setup( 'zope.interface>=4.1.3', 'protego>=0.1.15', 'itemadapter>=0.1.0', - 'typing_extensions>=3.7' + 'typing_extensions>=3.7', + 'h2>=3.2.0' ], extras_require=extras_require, ) \ No newline at end of file diff --git a/tox.ini b/tox.ini index ada211b3c..bc6314a2f 100644 --- a/tox.ini +++ b/tox.ini @@ -84,6 +84,7 @@ deps = # Extras botocore==1.3.23 Pillow==3.4.2 + h2==3.2.0 [testenv:extra-deps] deps = From ec06cf79a6a7264ec3e32d7de8c4e305c2afa05e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 6 Jul 2020 10:47:11 +0200 Subject: [PATCH 153/568] Update tests/CrawlerRunner/ip_address.py Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> --- tests/CrawlerRunner/ip_address.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index ea75bc3c9..b8254afdf 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -15,8 +15,7 @@ from tests.mockserver import MockServer, MockDNSServer # https://stackoverflow.com/a/32784190 def createResolver(servers=None, resolvconf=None, hosts=None): if hosts is None: - hosts = (b'/etc/hosts' if platform.getType() == 'posix' - else r'c:\windows\hosts') + hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts' theResolver = Resolver(resolvconf, servers) hostResolver = hostsModule.Resolver(hosts) L = [hostResolver, cache.CacheResolver(), theResolver] From 17aec5944cab33b3cdcd497d2362cacbf7773e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 6 Jul 2020 10:47:25 +0200 Subject: [PATCH 154/568] Update tests/CrawlerRunner/ip_address.py Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> --- tests/CrawlerRunner/ip_address.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index b8254afdf..3f9738798 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -18,8 +18,8 @@ def createResolver(servers=None, resolvconf=None, hosts=None): hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts' theResolver = Resolver(resolvconf, servers) hostResolver = hostsModule.Resolver(hosts) - L = [hostResolver, cache.CacheResolver(), theResolver] - return resolve.ResolverChain(L) + chain = [hostResolver, cache.CacheResolver(), theResolver] + return resolve.ResolverChain(chain) class LocalhostSpider(Spider): From f1020e0e6af064ab31b812c25bda6b0f08827222 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Mon, 6 Jul 2020 15:40:53 +0300 Subject: [PATCH 155/568] Tiny changes --- scrapy/extensions/feedexport.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e15c1a09c..21177b1b0 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -305,7 +305,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri + :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ storage = self._get_storage(uri) file = storage.open(spider) @@ -372,13 +372,13 @@ class FeedExporter: def _settings_are_valid(self): """ If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain - %(batch_time)s or %(batch_id)s to distinguish different files of partial output + %(batch_time)s or %(batch_id)d to distinguish different files of partial output """ for uri_template, values in self.feeds.items(): if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): logger.error( - '%(batch_time)s or %(batch_id) must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' - 'or FEEDS.batch_item_count is specified and greater than 0. For more info see:' + '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT ' + 'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: ' 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' ''.format(uri_template) ) From 54e4228c3a22164b79db36a29a5e2d64391b592a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 6 Jul 2020 14:10:45 -0300 Subject: [PATCH 156/568] refactor: use protocol - H2ClientProtocol.close_stream - Fix and add missing type hints - More adjustments - Rename stream id generator - Simplify decrement --- scrapy/core/http2/protocol.py | 92 +++++++++++++++--------------- scrapy/core/http2/stream.py | 103 ++++++++++++++++------------------ 2 files changed, 93 insertions(+), 102 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index a3dfdb76e..2f177656d 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -11,32 +11,34 @@ from h2.events import ( StreamEnded, StreamReset, WindowUpdated ) from h2.exceptions import ProtocolError +from twisted.internet.defer import Deferred from twisted.internet.protocol import connectionDone, Protocol from twisted.internet.ssl import Certificate +from twisted.python.failure import Failure from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request + logger = logging.getLogger(__name__) class H2ClientProtocol(Protocol): - def __init__(self): + def __init__(self) -> None: config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) # ID of the next request stream - # Following the convention made by hyper-h2 each client ID - # will be odd. - self.stream_id_count = itertools.count(start=1, step=2) + # Following the convention made by hyper-h2 all IDs will be odd + self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs self.streams: Dict[int, Stream] = {} # If requests are received before connection is made we keep # all requests in a pool and send them as the connection is made - self._pending_request_stream_pool = deque() + self._pending_request_stream_pool: deque = deque() # Counter to keep track of opened stream. This counter # is used to make sure that not more than MAX_CONCURRENT_STREAMS @@ -48,15 +50,15 @@ class H2ClientProtocol(Protocol): # We pass this instance to the streams ResponseFailed() failure self._protocol_error: Optional[ProtocolError] = None - self._metadata: H2ConnectionMetadataDict = { + self.metadata: H2ConnectionMetadataDict = { 'certificate': None, 'ip_address': None, 'hostname': None, - 'port': None + 'port': None, } @property - def is_connected(self): + def is_connected(self) -> bool: """Boolean to keep track of the connection status. This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection @@ -75,7 +77,7 @@ class H2ClientProtocol(Protocol): self.conn.remote_settings.max_concurrent_streams ) - def _send_pending_requests(self): + def _send_pending_requests(self) -> None: """Initiate all pending requests from the deque following FIFO We make sure that at any time {allowed_max_concurrent_streams} streams are active. @@ -89,37 +91,33 @@ class H2ClientProtocol(Protocol): stream = self._pending_request_stream_pool.popleft() stream.initiate_request() - def _stream_close_cb(self, stream_id: int): - """Called when stream is closed completely + def pop_stream(self, stream_id: int) -> Stream: + """Perform cleanup when a stream is closed """ - self.streams.pop(stream_id) + stream = self.streams.pop(stream_id) self._active_streams -= 1 self._send_pending_requests() + return stream - def _new_stream(self, request: Request): + def _new_stream(self, request: Request) -> Stream: """Instantiates a new Stream object """ - stream_id = next(self.stream_id_count) - stream = Stream( - stream_id=stream_id, + stream_id=next(self._stream_id_generator), request=request, - connection=self.conn, - conn_metadata=self._metadata, - cb_close=self._stream_close_cb + protocol=self, ) - self.streams[stream.stream_id] = stream return stream - def _write_to_transport(self): + def _write_to_transport(self) -> None: """ Write data to the underlying transport connection from the HTTP2 connection instance if any """ data = self.conn.data_to_send() self.transport.write(data) - def request(self, request: Request): + def request(self, request: Request) -> Deferred: if not isinstance(request, Request): raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}') @@ -130,20 +128,20 @@ class H2ClientProtocol(Protocol): self._pending_request_stream_pool.append(stream) return d - def connectionMade(self): + def connectionMade(self) -> None: """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ destination = self.transport.getPeer() logger.debug('Connection made to {}'.format(destination)) - self._metadata['ip_address'] = ipaddress.ip_address(destination.host) - self._metadata['port'] = destination.port - self._metadata['hostname'] = self.transport.transport.addr[0] + self.metadata['ip_address'] = ipaddress.ip_address(destination.host) + self.metadata['port'] = destination.port + self.metadata['hostname'] = self.transport.transport.addr[0] self.conn.initiate_connection() self._write_to_transport() - def dataReceived(self, data): + def dataReceived(self, data: bytes) -> None: try: events = self.conn.receive_data(data) self._handle_events(events) @@ -158,32 +156,30 @@ class H2ClientProtocol(Protocol): finally: self._write_to_transport() - def connectionLost(self, reason=connectionDone): + def connectionLost(self, reason: Failure = connectionDone) -> None: """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ - # Pop all streams which were pending and were not yet started - # NOTE: Stream.close() pops the element from the streams dictionary - # which raises `RuntimeError: dictionary changed size during iteration` - # Hence, we copy the streams into a list. - for stream in list(self.streams.values()): + for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error) + stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error, from_protocol=True) else: - stream.close(StreamCloseReason.INACTIVE) + stream.close(StreamCloseReason.INACTIVE, from_protocol=True) + self._active_streams -= len(self.streams) + self.streams.clear() + self._send_pending_requests() self.conn.close_connection() if not reason.check(connectionDone): logger.warning("Connection lost with reason " + str(reason)) - def _handle_events(self, events): + def _handle_events(self, events: list) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler Arguments: - events {list} -- A list of events that the remote peer - triggered by sending data + events -- A list of events that the remote peer triggered by sending data """ for event in events: if isinstance(event, DataReceived): @@ -202,27 +198,29 @@ class H2ClientProtocol(Protocol): logger.debug('Received unhandled event {}'.format(event)) # Event handler functions starts here - def data_received(self, event: DataReceived): + def data_received(self, event: DataReceived) -> None: self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) - def response_received(self, event: ResponseReceived): + def response_received(self, event: ResponseReceived) -> None: self.streams[event.stream_id].receive_headers(event.headers) - def settings_acknowledged(self, event: SettingsAcknowledged): + def settings_acknowledged(self, event: SettingsAcknowledged) -> None: # Send off all the pending requests as now we have # established a proper HTTP/2 connection self._send_pending_requests() # Update certificate when our HTTP/2 connection is established - self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) + self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) - def stream_ended(self, event: StreamEnded): - self.streams[event.stream_id].close(StreamCloseReason.ENDED) + def stream_ended(self, event: StreamEnded) -> None: + stream = self.pop_stream(event.stream_id) + stream.close(StreamCloseReason.ENDED, from_protocol=True) - def stream_reset(self, event: StreamReset): - self.streams[event.stream_id].close(StreamCloseReason.RESET) + def stream_reset(self, event: StreamReset) -> None: + stream = self.pop_stream(event.stream_id) + stream.close(StreamCloseReason.RESET, from_protocol=True) - def window_updated(self, event: WindowUpdated): + def window_updated(self, event: WindowUpdated) -> None: if event.stream_id != 0: self.streams[event.stream_id].receive_window_update() else: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 1c856ff68..77cfbcfbf 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,22 +1,27 @@ import logging from enum import Enum from io import BytesIO -from typing import Callable, List +from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse -from h2.connection import H2Connection from h2.errors import ErrorCodes from h2.exceptions import StreamClosedError +from hpack import HeaderTuple from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from scrapy.core.http2.types import H2ConnectionMetadataDict, H2ResponseDict +from scrapy.core.http2.types import H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes + +if TYPE_CHECKING: + from scrapy.core.http2.protocol import H2ClientProtocol + + logger = logging.getLogger(__name__) @@ -31,12 +36,12 @@ class InactiveStreamClosed(ConnectionClosed): class InvalidHostname(Exception): - def __init__(self, request: Request, expected_hostname, expected_netloc): + def __init__(self, request: Request, expected_hostname: Optional[str], expected_netloc: Optional[str]) -> None: self.request = request self.expected_hostname = expected_hostname self.expected_netloc = expected_netloc - def __str__(self): + def __str__(self) -> str: return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}' @@ -80,28 +85,20 @@ class Stream: self, stream_id: int, request: Request, - connection: H2Connection, - conn_metadata: H2ConnectionMetadataDict, - cb_close: Callable[[int], None], + protocol: "H2ClientProtocol", download_maxsize: int = 0, download_warnsize: int = 0, fail_on_data_loss: bool = True - ): + ) -> None: """ Arguments: - stream_id -- For one HTTP/2 connection each stream is - uniquely identified by a single integer - request -- HTTP request - connection -- HTTP/2 connection this stream belongs to. - conn_metadata -- Reference to dictionary having metadata of HTTP/2 connection - cb_close -- Method called when this stream is closed - to notify the TCP connection instance. + stream_id -- Unique identifier for the stream within a single HTTP/2 connection + request -- The HTTP request associated to the stream + protocol -- Parent H2ClientProtocol instance """ - self.stream_id = stream_id - self._request = request - self._conn = connection - self._conn_metadata = conn_metadata - self._cb_close = cb_close + self.stream_id: int = stream_id + self._request: Request = request + self._protocol: "H2ClientProtocol" = protocol self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) @@ -132,7 +129,7 @@ class Stream: self._response: H2ResponseDict = { 'body': BytesIO(), 'flow_controlled_size': 0, - 'headers': Headers({}) + 'headers': Headers({}), } def _cancel(_): @@ -145,7 +142,7 @@ class Stream: self._deferred_response = Deferred(_cancel) - def __str__(self): + def __str__(self) -> str: return f'Stream(id={self.stream_id!r})' __repr__ = __str__ @@ -169,12 +166,9 @@ class Stream: and not self._reached_warnsize ) - def get_response(self): + def get_response(self) -> Deferred: """Simply return a Deferred which fires when response from the asynchronous request is available - - Returns: - Deferred -- Calls the callback passing the response """ return self._deferred_response @@ -182,12 +176,12 @@ class Stream: # Make sure that we are sending the request to the correct URL url = urlparse(self._request.url) return ( - url.netloc == self._conn_metadata['hostname'] - or url.netloc == f'{self._conn_metadata["hostname"]}:{self._conn_metadata["port"]}' - or url.netloc == f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}' + url.netloc == self._protocol.metadata['hostname'] + or url.netloc == f'{self._protocol.metadata["hostname"]}:{self._protocol.metadata["port"]}' + or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' ) - def _get_request_headers(self): + def _get_request_headers(self) -> List[Tuple[str, str]]: url = urlparse(self._request.url) path = url.path @@ -207,10 +201,10 @@ class Stream: return headers - def initiate_request(self): + def initiate_request(self) -> None: if self.check_request_url(): headers = self._get_request_headers() - self._conn.send_headers(self.stream_id, headers, end_stream=False) + self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False) self.request_sent = True self.send_data() else: @@ -218,7 +212,7 @@ class Stream: # Note that we have not sent any headers self.close(StreamCloseReason.INVALID_HOSTNAME) - def send_data(self): + def send_data(self) -> None: """Called immediately after the headers are sent. Here we send all the data as part of the request. @@ -233,10 +227,10 @@ class Stream: raise StreamClosedError(self.stream_id) # Firstly, check what the flow control window is for current stream. - window_size = self._conn.local_flow_control_window(stream_id=self.stream_id) + window_size = self._protocol.conn.local_flow_control_window(stream_id=self.stream_id) # Next, check what the maximum frame size is. - max_frame_size = self._conn.max_outbound_frame_size + max_frame_size = self._protocol.conn.max_outbound_frame_size # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. @@ -249,7 +243,7 @@ class Stream: data_chunk_start_id = self.content_length - self.remaining_content_length data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size] - self._conn.send_data(self.stream_id, data_chunk, end_stream=False) + self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False) bytes_to_send_size = bytes_to_send_size - chunk_size self.remaining_content_length = self.remaining_content_length - chunk_size @@ -258,12 +252,12 @@ class Stream: # End the stream if no more data needs to be send if self.remaining_content_length == 0: - self._conn.end_stream(self.stream_id) + self._protocol.conn.end_stream(self.stream_id) # Q. What about the rest of the data? # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame - def receive_window_update(self): + def receive_window_update(self) -> None: """Flow control window size was changed. Send data that earlier could not be sent as we were blocked behind the flow control. @@ -271,7 +265,7 @@ class Stream: if self.remaining_content_length and not self.stream_closed_server and self.request_sent: self.send_data() - def receive_data(self, data: bytes, flow_controlled_length: int): + def receive_data(self, data: bytes, flow_controlled_length: int) -> None: self._response['body'].write(data) self._response['flow_controlled_size'] += flow_controlled_length @@ -289,12 +283,12 @@ class Stream: logger.warning(warning_msg) # Acknowledge the data received - self._conn.acknowledge_received_data( + self._protocol.conn.acknowledge_received_data( self._response['flow_controlled_size'], self.stream_id ) - def receive_headers(self, headers): + def receive_headers(self, headers: List[HeaderTuple]) -> None: for name, value in headers: self._response['headers'][name] = value @@ -312,7 +306,7 @@ class Stream: ) logger.warning(warning_msg) - def reset_stream(self, reason=StreamCloseReason.RESET): + def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None: """Close this stream by sending a RST_FRAME to the remote peer""" if self.stream_closed_local: raise StreamClosedError(self.stream_id) @@ -321,7 +315,7 @@ class Stream: self._response['body'].truncate(0) self.stream_closed_local = True - self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) + self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) self.close(reason) def _is_data_lost(self) -> bool: @@ -332,11 +326,8 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason, error: Exception = None): + def close(self, reason: StreamCloseReason, error: Optional[Exception] = None, from_protocol: bool = False) -> None: """Based on the reason sent we will handle each case. - - Arguments: - reason -- One if StreamCloseReason """ if self.stream_closed_server: raise StreamClosedError(self.stream_id) @@ -344,7 +335,9 @@ class Stream: if not isinstance(reason, StreamCloseReason): raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}') - self._cb_close(self.stream_id) + if not from_protocol: + self._protocol.pop_stream(self.stream_id) + self.stream_closed_server = True flags = None @@ -392,11 +385,11 @@ class Stream: elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( self._request, - self._conn_metadata['hostname'], - f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}' + self._protocol.metadata['hostname'], + f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' )) - def _fire_response_deferred(self, flags: List[str] = None): + def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None: """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" @@ -405,7 +398,7 @@ class Stream: response_cls = responsetypes.from_args( headers=self._response['headers'], url=self._request.url, - body=body + body=body, ) response = response_cls( @@ -415,8 +408,8 @@ class Stream: body=body, request=self._request, flags=flags, - certificate=self._conn_metadata['certificate'], - ip_address=self._conn_metadata['ip_address'] + certificate=self._protocol.metadata['certificate'], + ip_address=self._protocol.metadata['ip_address'], ) self._deferred_response.callback(response) From 770a8127e8e76d95243c1d586b4bb6113a38870a Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Tue, 7 Jul 2020 15:23:29 +0530 Subject: [PATCH 157/568] Added basic `scrapy check` tests --- tests/test_command_check.py | 96 +++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/test_command_check.py diff --git a/tests/test_command_check.py b/tests/test_command_check.py new file mode 100644 index 000000000..52005a4c5 --- /dev/null +++ b/tests/test_command_check.py @@ -0,0 +1,96 @@ +from os.path import join, abspath + +from tests.test_commands import CommandTest + + +class CheckCommandTest(CommandTest): + + command = 'check' + + def setUp(self): + super(CheckCommandTest, self).setUp() + self.spider_name = 'check_spider' + self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py')) + + def _write_contract(self, contracts, parse_def): + with open(self.spider, 'w') as file: + file.write(f""" +import scrapy + +class CheckSpider(scrapy.Spider): + name = '{self.spider_name}' + start_urls = ['http://example.com'] + + def parse(self, response, **cb_kwargs): + \"\"\" + @url http://www.amazon.com/s?field-keywords=selfish+gene + {contracts} + \"\"\" + {parse_def} + """) + + def _test_contract(self, contracts='', parse_def='pass'): + self._write_contract(contracts, parse_def) + p, out, err = self.proc('check') + self.assertIn('OK', err) + self.assertEqual(p.returncode, 0) + + def test_check_returns_requests_contract(self): + contracts = """ + @returns requests 1 + """ + parse_def = """ + yield scrapy.Request(url='http://next-url.com') + """ + self._test_contract(contracts, parse_def) + + def test_check_returns_items_contract(self): + contracts = """ + @returns items 1 + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + """ + self._test_contract(contracts, parse_def) + + def test_check_cb_kwargs_contract(self): + contracts = """ + @cb_kwargs {"arg1": "val1", "arg2": "val2"} + """ + parse_def = """ + if len(cb_kwargs.items()) == 0: + raise Exception("Callback args not set") + """ + self._test_contract(contracts, parse_def) + + def test_check_scrapes_contract(self): + contracts = """ + @scrapes key1 key2 + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + """ + self._test_contract(contracts, parse_def) + + def test_check_all_default_contracts(self): + contracts = """ + @returns items 1 + @returns requests 1 + @scrapes key1 key2 + @cb_kwargs {"arg1": "val1", "arg2": "val2"} + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + yield scrapy.Request(url='http://next-url.com') + if len(cb_kwargs.items()) == 0: + raise Exception("Callback args not set") + """ + self._test_contract(contracts, parse_def) + + def test_SCRAPY_CHECK_set(self): + parse_def = """ + import os + if not os.environ.get('SCRAPY_CHECK'): + raise Exception('SCRAPY_CHECK not set') + """ + self._test_contract(parse_def=parse_def) From d014840672820b3970282f31a51b9ff24cd46bd3 Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Tue, 7 Jul 2020 15:24:33 +0530 Subject: [PATCH 158/568] Ignore flake8 E501 for `scrapy check` tests` --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index bae68cd3a..97320a008 100644 --- a/pytest.ini +++ b/pytest.ini @@ -173,6 +173,7 @@ flake8-ignore = tests/pipelines.py F841 E226 tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 + tests/test_command_check.py E501 tests/test_command_fetch.py E501 tests/test_command_parse.py E501 E128 E303 E226 tests/test_command_shell.py E501 E128 From 1c40dfa7408026bc9ae831000a8614e8f4a9dd0d Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Tue, 7 Jul 2020 00:44:09 +0530 Subject: [PATCH 159/568] fix: handle CONNECTION_LOST & RESET separately --- scrapy/core/http2/protocol.py | 19 ++++++++++++------- scrapy/core/http2/stream.py | 18 ++++++++++++------ scrapy/utils/log.py | 6 +++--- tests/test_http2_client_protocol.py | 2 +- tox.ini | 2 +- 5 files changed, 29 insertions(+), 18 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 2f177656d..55dbcabec 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -20,7 +20,6 @@ from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request - logger = logging.getLogger(__name__) @@ -30,7 +29,8 @@ class H2ClientProtocol(Protocol): self.conn = H2Connection(config=config) # ID of the next request stream - # Following the convention made by hyper-h2 all IDs will be odd + # Following the convention - 'Streams initiated by a client MUST + # use odd-numbered stream identifiers' (RFC 7540) self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs @@ -160,20 +160,25 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ + errors = [] + if not reason.check(connectionDone): + logger.warning("Connection lost with reason " + str(reason)) + errors.append(reason) + + if self._protocol_error: + errors.append(self._protocol_error) + for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error, from_protocol=True) + stream.close(StreamCloseReason.CONNECTION_LOST, errors, from_protocol=True) else: stream.close(StreamCloseReason.INACTIVE, from_protocol=True) self._active_streams -= len(self.streams) self.streams.clear() - self._send_pending_requests() + self._pending_request_stream_pool.clear() self.conn.close_connection() - if not reason.check(connectionDone): - logger.warning("Connection lost with reason " + str(reason)) - def _handle_events(self, events: list) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 77cfbcfbf..8b66d4b85 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -17,11 +17,9 @@ from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes - if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol - logger = logging.getLogger(__name__) @@ -326,7 +324,12 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason, error: Optional[Exception] = None, from_protocol: bool = False) -> None: + def close( + self, + reason: StreamCloseReason, + errors: Optional[List[Exception]] = None, + from_protocol: bool = False + ) -> None: """Based on the reason sent we will handle each case. """ if self.stream_closed_server: @@ -374,11 +377,14 @@ class Stream: self._response['headers'][':status'] = '499' self._fire_response_deferred() - elif reason in (StreamCloseReason.RESET, StreamCloseReason.CONNECTION_LOST): + elif reason is StreamCloseReason.RESET: self._deferred_response.errback(ResponseFailed([ - error if error else Failure() + Failure(f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM') ])) + elif reason is StreamCloseReason.CONNECTION_LOST: + self._deferred_response.errback(ResponseFailed(errors)) + elif reason is StreamCloseReason.INACTIVE: self._deferred_response.errback(InactiveStreamClosed(self._request)) @@ -403,7 +409,7 @@ class Stream: response = response_cls( url=self._request.url, - status=self._response['headers'][':status'], + status=int(self._response['headers'][':status']), headers=self._response['headers'], body=body, request=self._request, diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 4e2671478..9d59fdd68 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -46,15 +46,15 @@ DEFAULT_LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'loggers': { + 'hpack': { + 'level': 'ERROR', + }, 'scrapy': { 'level': 'DEBUG', }, 'twisted': { 'level': 'ERROR', }, - 'hpack': { - 'level': 'ERROR', - }, } } diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 98dc98a0f..9efca5267 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -474,7 +474,7 @@ class Https2ClientProtocolTestCase(TestCase): # Close the connection now to fire all the extra 10 requests errback # with InactiveStreamClosed - self.client.transport.abortConnection() + self.client.transport.loseConnection() return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) diff --git a/tox.ini b/tox.ini index bc6314a2f..be3b56e2d 100644 --- a/tox.ini +++ b/tox.ini @@ -83,8 +83,8 @@ deps = -rtests/requirements-py3.txt # Extras botocore==1.3.23 - Pillow==3.4.2 h2==3.2.0 + Pillow==3.4.2 [testenv:extra-deps] deps = From 79b4dfc53e431bdad31925aaf16831a0dde536ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 7 Jul 2020 14:07:04 +0200 Subject: [PATCH 160/568] Fix permission handling on project generation from template files --- scrapy/commands/startproject.py | 30 +--- .../project/module/__init__.py | 0 .../project/module/items.py.tmpl | 12 ++ .../project/module/middlewares.py.tmpl | 103 +++++++++++ .../project/module/pipelines.py.tmpl | 13 ++ .../project/module/settings.py.tmpl | 88 ++++++++++ .../project/module/spiders/__init__.py | 4 + .../read_only_templates/project/scrapy.cfg | 11 ++ .../read_only_templates/spiders/basic.tmpl | 10 ++ .../read_only_templates/spiders/crawl.tmpl | 20 +++ .../read_only_templates/spiders/csvfeed.tmpl | 20 +++ .../read_only_templates/spiders/xmlfeed.tmpl | 16 ++ tests/test_commands.py | 164 ++++++++++++++++++ 13 files changed, 467 insertions(+), 24 deletions(-) create mode 100644 tests/sample_data/read_only_templates/project/module/__init__.py create mode 100644 tests/sample_data/read_only_templates/project/module/items.py.tmpl create mode 100644 tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl create mode 100644 tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl create mode 100644 tests/sample_data/read_only_templates/project/module/settings.py.tmpl create mode 100644 tests/sample_data/read_only_templates/project/module/spiders/__init__.py create mode 100644 tests/sample_data/read_only_templates/project/scrapy.cfg create mode 100644 tests/sample_data/read_only_templates/spiders/basic.tmpl create mode 100644 tests/sample_data/read_only_templates/spiders/crawl.tmpl create mode 100644 tests/sample_data/read_only_templates/spiders/csvfeed.tmpl create mode 100644 tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 852281959..e702d7cdc 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,10 +1,10 @@ import re import os -import stat import string from importlib import import_module from os.path import join, exists, abspath from shutil import ignore_patterns, move, copy2, copystat +from stat import S_IWUSR as OWNER_WRITE_PERMISSION import scrapy from scrapy.commands import ScrapyCommand @@ -78,30 +78,12 @@ class Command(ScrapyCommand): self._copytree(srcname, dstname) else: copy2(srcname, dstname) + current_permissions = os.stat(dstname).st_mode + os.chmod(dstname, current_permissions | OWNER_WRITE_PERMISSION) + copystat(src, dst) - self._set_rw_permissions(dst) - - def _set_rw_permissions(self, path): - """ - Sets permissions of a directory tree to +rw and +rwx for folders. - This is necessary if the start template files come without write - permissions. - """ - mode_rw = (stat.S_IRUSR - | stat.S_IWUSR - | stat.S_IRGRP - | stat.S_IROTH) - - mode_x = (stat.S_IXUSR - | stat.S_IXGRP - | stat.S_IXOTH) - - os.chmod(path, mode_rw | mode_x) - for root, dirs, files in os.walk(path): - for dir in dirs: - os.chmod(join(root, dir), mode_rw | mode_x) - for file in files: - os.chmod(join(root, file), mode_rw) + current_permissions = os.stat(dst).st_mode + os.chmod(dst, current_permissions | OWNER_WRITE_PERMISSION) def run(self, args, opts): if len(args) not in (1, 2): diff --git a/tests/sample_data/read_only_templates/project/module/__init__.py b/tests/sample_data/read_only_templates/project/module/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/sample_data/read_only_templates/project/module/items.py.tmpl b/tests/sample_data/read_only_templates/project/module/items.py.tmpl new file mode 100644 index 000000000..88a18331c --- /dev/null +++ b/tests/sample_data/read_only_templates/project/module/items.py.tmpl @@ -0,0 +1,12 @@ +# Define here the models for your scraped items +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/items.html + +import scrapy + + +class ${ProjectName}Item(scrapy.Item): + # define the fields for your item here like: + # name = scrapy.Field() + pass diff --git a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl new file mode 100644 index 000000000..bd09890fe --- /dev/null +++ b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl @@ -0,0 +1,103 @@ +# Define here the models for your spider middleware +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals + +# useful for handling different item types with a single interface +from itemadapter import is_item, ItemAdapter + + +class ${ProjectName}SpiderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, or item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Request or item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class ${ProjectName}DownloaderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl new file mode 100644 index 000000000..e845f43e9 --- /dev/null +++ b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl @@ -0,0 +1,13 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html + + +# useful for handling different item types with a single interface +from itemadapter import ItemAdapter + + +class ${ProjectName}Pipeline: + def process_item(self, item, spider): + return item diff --git a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl new file mode 100644 index 000000000..a414b5fde --- /dev/null +++ b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl @@ -0,0 +1,88 @@ +# Scrapy settings for $project_name project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +BOT_NAME = '$project_name' + +SPIDER_MODULES = ['$project_name.spiders'] +NEWSPIDER_MODULE = '$project_name.spiders' + + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = '$project_name (+http://www.yourdomain.com)' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = True + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +#CONCURRENT_REQUESTS = 32 + +# Configure a delay for requests for the same website (default: 0) +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +#DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} + +# Enable or disable spider middlewares +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +#DOWNLOADER_MIDDLEWARES = { +# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, +#} + +# Enable or disable extensions +# See https://docs.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html +#ITEM_PIPELINES = { +# '$project_name.pipelines.${ProjectName}Pipeline': 300, +#} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py new file mode 100644 index 000000000..ebd689ac5 --- /dev/null +++ b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/tests/sample_data/read_only_templates/project/scrapy.cfg b/tests/sample_data/read_only_templates/project/scrapy.cfg new file mode 100644 index 000000000..1daeaa541 --- /dev/null +++ b/tests/sample_data/read_only_templates/project/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = ${project_name}.settings + +[deploy] +#url = http://localhost:6800/ +project = ${project_name} diff --git a/tests/sample_data/read_only_templates/spiders/basic.tmpl b/tests/sample_data/read_only_templates/spiders/basic.tmpl new file mode 100644 index 000000000..e9112bc95 --- /dev/null +++ b/tests/sample_data/read_only_templates/spiders/basic.tmpl @@ -0,0 +1,10 @@ +import scrapy + + +class $classname(scrapy.Spider): + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://$domain/'] + + def parse(self, response): + pass diff --git a/tests/sample_data/read_only_templates/spiders/crawl.tmpl b/tests/sample_data/read_only_templates/spiders/crawl.tmpl new file mode 100644 index 000000000..356496487 --- /dev/null +++ b/tests/sample_data/read_only_templates/spiders/crawl.tmpl @@ -0,0 +1,20 @@ +import scrapy +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import CrawlSpider, Rule + + +class $classname(CrawlSpider): + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://$domain/'] + + rules = ( + Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), + ) + + def parse_item(self, response): + item = {} + #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get() + #item['name'] = response.xpath('//div[@id="name"]').get() + #item['description'] = response.xpath('//div[@id="description"]').get() + return item diff --git a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl new file mode 100644 index 000000000..cbcbe9e2c --- /dev/null +++ b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl @@ -0,0 +1,20 @@ +from scrapy.spiders import CSVFeedSpider + + +class $classname(CSVFeedSpider): + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://$domain/feed.csv'] + # headers = ['id', 'name', 'description', 'image_link'] + # delimiter = '\t' + + # Do any adaptations you need here + #def adapt_response(self, response): + # return response + + def parse_row(self, response, row): + i = {} + #i['url'] = row['url'] + #i['name'] = row['name'] + #i['description'] = row['description'] + return i diff --git a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl new file mode 100644 index 000000000..5aa2aa8b0 --- /dev/null +++ b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl @@ -0,0 +1,16 @@ +from scrapy.spiders import XMLFeedSpider + + +class $classname(XMLFeedSpider): + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://$domain/feed.xml'] + iterator = 'iternodes' # you can change this; see the docs + itertag = 'item' # change it accordingly + + def parse_node(self, response, selector): + item = {} + #item['url'] = selector.select('url').get() + #item['name'] = selector.select('name').get() + #item['description'] = selector.select('description').get() + return item diff --git a/tests/test_commands.py b/tests/test_commands.py index 24a341759..8336c8759 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -6,7 +6,9 @@ import subprocess import sys import tempfile from contextlib import contextmanager +from itertools import chain from os.path import exists, join, abspath +from pathlib import Path from shutil import rmtree, copytree from tempfile import mkdtemp from threading import Timer @@ -15,6 +17,7 @@ from twisted.trial import unittest import scrapy from scrapy.commands import ScrapyCommand +from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv @@ -119,8 +122,34 @@ class StartprojectTest(ProjectTest): self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params')) +def get_permissions_dict(path, renamings=None, ignore=None): + renamings = renamings or tuple() + permissions_dict = { + '.': os.stat(path).st_mode, + } + for root, dirs, files in os.walk(path): + nodes = list(chain(dirs, files)) + if ignore: + ignored_names = ignore(root, nodes) + nodes = [node for node in nodes + if node not in ignored_names] + for node in nodes: + absolute_path = os.path.join(root, node) + relative_path = os.path.relpath(absolute_path, path) + for search_string, replacement in renamings: + relative_path = relative_path.replace( + search_string, + replacement + ) + permissions = os.stat(absolute_path).st_mode + permissions_dict[relative_path] = permissions + return permissions_dict + + class StartprojectTemplatesTest(ProjectTest): + maxDiff = None + def setUp(self): super(StartprojectTemplatesTest, self).setUp() self.tmpl = join(self.temp_path, 'templates') @@ -139,6 +168,141 @@ class StartprojectTemplatesTest(ProjectTest): self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) + def test_startproject_permissions_from_writable(self): + """Check that generated files have the right permissions when the + template folder has the same permissions as in the project, i.e. + everything is writable.""" + scrapy_path = scrapy.__path__[0] + templates_dir = os.path.join(scrapy_path, 'templates', 'project') + project_name = 'startproject1' + renamings = ( + ('module', project_name), + ('.tmpl', ''), + ) + expected_permissions = get_permissions_dict( + templates_dir, + renamings, + IGNORE, + ) + + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'startproject', + project_name, + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = os.path.join(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + self.assertEqual(actual_permissions, expected_permissions) + + def test_startproject_permissions_from_read_only(self): + """Check that generated files have the right permissions when the + template folder has been made read-only, which is something that some + systems do. + + See https://github.com/scrapy/scrapy/pull/4604 + """ + scrapy_path = scrapy.__path__[0] + templates_dir = os.path.join(scrapy_path, 'templates', 'project') + project_name = 'startproject2' + renamings = ( + ('module', project_name), + ('.tmpl', ''), + ) + expected_permissions = get_permissions_dict( + templates_dir, + renamings, + IGNORE, + ) + + tests_path = os.path.dirname(__file__) + read_only_templates_dir = os.path.join( + tests_path, 'sample_data', 'read_only_templates' + ) + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'startproject', + project_name, + '--set', + 'TEMPLATES_DIR={}'.format(read_only_templates_dir), + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = os.path.join(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + self.assertEqual(actual_permissions, expected_permissions) + + def test_startproject_permissions_unchanged_in_destination(self): + """Check that pre-existing folders and files in the destination folder + do not see their permissions modified.""" + scrapy_path = scrapy.__path__[0] + templates_dir = os.path.join(scrapy_path, 'templates', 'project') + project_name = 'startproject3' + renamings = ( + ('module', project_name), + ('.tmpl', ''), + ) + expected_permissions = get_permissions_dict( + templates_dir, + renamings, + IGNORE, + ) + + destination = mkdtemp() + project_dir = os.path.join(destination, project_name) + + existing_nodes = { + oct(permissions)[2:] + extension: permissions + for extension in ('', '.d') + for permissions in ( + 0o444, 0o555, 0o644, 0o666, 0o755, 0o777, + ) + } + os.mkdir(project_dir) + project_dir_path = Path(project_dir) + for node, permissions in existing_nodes.items(): + path = project_dir_path / node + if node.endswith('.d'): + path.mkdir(mode=permissions) + else: + path.touch(mode=permissions) + expected_permissions[node] = path.stat().st_mode + + process = subprocess.Popen( + ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'startproject', + project_name, + '.', + ), + cwd=project_dir, + env=self.env, + ) + process.wait() + + actual_permissions = get_permissions_dict(project_dir) + + self.assertEqual(actual_permissions, expected_permissions) + class CommandTest(ProjectTest): From a3afff4a0e1b25903d1de5c5501846d1f1288d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 7 Jul 2020 14:11:02 +0200 Subject: [PATCH 161/568] Fix style issue --- tests/test_commands.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 8336c8759..bd799817d 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -131,8 +131,7 @@ def get_permissions_dict(path, renamings=None, ignore=None): nodes = list(chain(dirs, files)) if ignore: ignored_names = ignore(root, nodes) - nodes = [node for node in nodes - if node not in ignored_names] + nodes = [node for node in nodes if node not in ignored_names] for node in nodes: absolute_path = os.path.join(root, node) relative_path = os.path.relpath(absolute_path, path) From e1450799ce2dfa32e248cb0b4069668ee5e6f4ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 7 Jul 2020 14:11:37 +0200 Subject: [PATCH 162/568] Remove debug test case variable --- tests/test_commands.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index bd799817d..c25495d16 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -147,8 +147,6 @@ def get_permissions_dict(path, renamings=None, ignore=None): class StartprojectTemplatesTest(ProjectTest): - maxDiff = None - def setUp(self): super(StartprojectTemplatesTest, self).setUp() self.tmpl = join(self.temp_path, 'templates') From ca77ca1f751a614b0c9394d1e8c6d0b9cfcdd957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 7 Jul 2020 14:44:03 +0200 Subject: [PATCH 163/568] Generate read-only files on the fly --- scrapy/commands/startproject.py | 13 ++- .../project/module/__init__.py | 0 .../project/module/items.py.tmpl | 12 -- .../project/module/middlewares.py.tmpl | 103 ------------------ .../project/module/pipelines.py.tmpl | 13 --- .../project/module/settings.py.tmpl | 88 --------------- .../project/module/spiders/__init__.py | 4 - .../read_only_templates/project/scrapy.cfg | 11 -- .../read_only_templates/spiders/basic.tmpl | 10 -- .../read_only_templates/spiders/crawl.tmpl | 20 ---- .../read_only_templates/spiders/csvfeed.tmpl | 20 ---- .../read_only_templates/spiders/xmlfeed.tmpl | 16 --- tests/test_commands.py | 31 ++++-- 13 files changed, 28 insertions(+), 313 deletions(-) delete mode 100644 tests/sample_data/read_only_templates/project/module/__init__.py delete mode 100644 tests/sample_data/read_only_templates/project/module/items.py.tmpl delete mode 100644 tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl delete mode 100644 tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl delete mode 100644 tests/sample_data/read_only_templates/project/module/settings.py.tmpl delete mode 100644 tests/sample_data/read_only_templates/project/module/spiders/__init__.py delete mode 100644 tests/sample_data/read_only_templates/project/scrapy.cfg delete mode 100644 tests/sample_data/read_only_templates/spiders/basic.tmpl delete mode 100644 tests/sample_data/read_only_templates/spiders/crawl.tmpl delete mode 100644 tests/sample_data/read_only_templates/spiders/csvfeed.tmpl delete mode 100644 tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e702d7cdc..eccc2a3e1 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -20,7 +20,12 @@ TEMPLATES_TO_RENDER = ( ('${project_name}', 'middlewares.py.tmpl'), ) -IGNORE = ignore_patterns('*.pyc', '.svn') +IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn') + + +def _make_writable(path): + current_permissions = os.stat(path).st_mode + os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) class Command(ScrapyCommand): @@ -78,12 +83,10 @@ class Command(ScrapyCommand): self._copytree(srcname, dstname) else: copy2(srcname, dstname) - current_permissions = os.stat(dstname).st_mode - os.chmod(dstname, current_permissions | OWNER_WRITE_PERMISSION) + _make_writable(dstname) copystat(src, dst) - current_permissions = os.stat(dst).st_mode - os.chmod(dst, current_permissions | OWNER_WRITE_PERMISSION) + _make_writable(dst) def run(self, args, opts): if len(args) not in (1, 2): diff --git a/tests/sample_data/read_only_templates/project/module/__init__.py b/tests/sample_data/read_only_templates/project/module/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/sample_data/read_only_templates/project/module/items.py.tmpl b/tests/sample_data/read_only_templates/project/module/items.py.tmpl deleted file mode 100644 index 88a18331c..000000000 --- a/tests/sample_data/read_only_templates/project/module/items.py.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -# Define here the models for your scraped items -# -# See documentation in: -# https://docs.scrapy.org/en/latest/topics/items.html - -import scrapy - - -class ${ProjectName}Item(scrapy.Item): - # define the fields for your item here like: - # name = scrapy.Field() - pass diff --git a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl deleted file mode 100644 index bd09890fe..000000000 --- a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl +++ /dev/null @@ -1,103 +0,0 @@ -# Define here the models for your spider middleware -# -# See documentation in: -# https://docs.scrapy.org/en/latest/topics/spider-middleware.html - -from scrapy import signals - -# useful for handling different item types with a single interface -from itemadapter import is_item, ItemAdapter - - -class ${ProjectName}SpiderMiddleware: - # Not all methods need to be defined. If a method is not defined, - # scrapy acts as if the spider middleware does not modify the - # passed objects. - - @classmethod - def from_crawler(cls, crawler): - # This method is used by Scrapy to create your spiders. - s = cls() - crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) - return s - - def process_spider_input(self, response, spider): - # Called for each response that goes through the spider - # middleware and into the spider. - - # Should return None or raise an exception. - return None - - def process_spider_output(self, response, result, spider): - # Called with the results returned from the Spider, after - # it has processed the response. - - # Must return an iterable of Request, or item objects. - for i in result: - yield i - - def process_spider_exception(self, response, exception, spider): - # Called when a spider or process_spider_input() method - # (from other spider middleware) raises an exception. - - # Should return either None or an iterable of Request or item objects. - pass - - def process_start_requests(self, start_requests, spider): - # Called with the start requests of the spider, and works - # similarly to the process_spider_output() method, except - # that it doesn’t have a response associated. - - # Must return only requests (not items). - for r in start_requests: - yield r - - def spider_opened(self, spider): - spider.logger.info('Spider opened: %s' % spider.name) - - -class ${ProjectName}DownloaderMiddleware: - # Not all methods need to be defined. If a method is not defined, - # scrapy acts as if the downloader middleware does not modify the - # passed objects. - - @classmethod - def from_crawler(cls, crawler): - # This method is used by Scrapy to create your spiders. - s = cls() - crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) - return s - - def process_request(self, request, spider): - # Called for each request that goes through the downloader - # middleware. - - # Must either: - # - return None: continue processing this request - # - or return a Response object - # - or return a Request object - # - or raise IgnoreRequest: process_exception() methods of - # installed downloader middleware will be called - return None - - def process_response(self, request, response, spider): - # Called with the response returned from the downloader. - - # Must either; - # - return a Response object - # - return a Request object - # - or raise IgnoreRequest - return response - - def process_exception(self, request, exception, spider): - # Called when a download handler or a process_request() - # (from other downloader middleware) raises an exception. - - # Must either: - # - return None: continue processing this exception - # - return a Response object: stops process_exception() chain - # - return a Request object: stops process_exception() chain - pass - - def spider_opened(self, spider): - spider.logger.info('Spider opened: %s' % spider.name) diff --git a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl deleted file mode 100644 index e845f43e9..000000000 --- a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -# Define your item pipelines here -# -# Don't forget to add your pipeline to the ITEM_PIPELINES setting -# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html - - -# useful for handling different item types with a single interface -from itemadapter import ItemAdapter - - -class ${ProjectName}Pipeline: - def process_item(self, item, spider): - return item diff --git a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl deleted file mode 100644 index a414b5fde..000000000 --- a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl +++ /dev/null @@ -1,88 +0,0 @@ -# Scrapy settings for $project_name project -# -# For simplicity, this file contains only settings considered important or -# commonly used. You can find more settings consulting the documentation: -# -# https://docs.scrapy.org/en/latest/topics/settings.html -# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html -# https://docs.scrapy.org/en/latest/topics/spider-middleware.html - -BOT_NAME = '$project_name' - -SPIDER_MODULES = ['$project_name.spiders'] -NEWSPIDER_MODULE = '$project_name.spiders' - - -# Crawl responsibly by identifying yourself (and your website) on the user-agent -#USER_AGENT = '$project_name (+http://www.yourdomain.com)' - -# Obey robots.txt rules -ROBOTSTXT_OBEY = True - -# Configure maximum concurrent requests performed by Scrapy (default: 16) -#CONCURRENT_REQUESTS = 32 - -# Configure a delay for requests for the same website (default: 0) -# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay -# See also autothrottle settings and docs -#DOWNLOAD_DELAY = 3 -# The download delay setting will honor only one of: -#CONCURRENT_REQUESTS_PER_DOMAIN = 16 -#CONCURRENT_REQUESTS_PER_IP = 16 - -# Disable cookies (enabled by default) -#COOKIES_ENABLED = False - -# Disable Telnet Console (enabled by default) -#TELNETCONSOLE_ENABLED = False - -# Override the default request headers: -#DEFAULT_REQUEST_HEADERS = { -# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', -# 'Accept-Language': 'en', -#} - -# Enable or disable spider middlewares -# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html -#SPIDER_MIDDLEWARES = { -# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, -#} - -# Enable or disable downloader middlewares -# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html -#DOWNLOADER_MIDDLEWARES = { -# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, -#} - -# Enable or disable extensions -# See https://docs.scrapy.org/en/latest/topics/extensions.html -#EXTENSIONS = { -# 'scrapy.extensions.telnet.TelnetConsole': None, -#} - -# Configure item pipelines -# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html -#ITEM_PIPELINES = { -# '$project_name.pipelines.${ProjectName}Pipeline': 300, -#} - -# Enable and configure the AutoThrottle extension (disabled by default) -# See https://docs.scrapy.org/en/latest/topics/autothrottle.html -#AUTOTHROTTLE_ENABLED = True -# The initial download delay -#AUTOTHROTTLE_START_DELAY = 5 -# The maximum download delay to be set in case of high latencies -#AUTOTHROTTLE_MAX_DELAY = 60 -# The average number of requests Scrapy should be sending in parallel to -# each remote server -#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 -# Enable showing throttling stats for every response received: -#AUTOTHROTTLE_DEBUG = False - -# Enable and configure HTTP caching (disabled by default) -# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings -#HTTPCACHE_ENABLED = True -#HTTPCACHE_EXPIRATION_SECS = 0 -#HTTPCACHE_DIR = 'httpcache' -#HTTPCACHE_IGNORE_HTTP_CODES = [] -#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py deleted file mode 100644 index ebd689ac5..000000000 --- a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# This package will contain the spiders of your Scrapy project -# -# Please refer to the documentation for information on how to create and manage -# your spiders. diff --git a/tests/sample_data/read_only_templates/project/scrapy.cfg b/tests/sample_data/read_only_templates/project/scrapy.cfg deleted file mode 100644 index 1daeaa541..000000000 --- a/tests/sample_data/read_only_templates/project/scrapy.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Automatically created by: scrapy startproject -# -# For more information about the [deploy] section see: -# https://scrapyd.readthedocs.io/en/latest/deploy.html - -[settings] -default = ${project_name}.settings - -[deploy] -#url = http://localhost:6800/ -project = ${project_name} diff --git a/tests/sample_data/read_only_templates/spiders/basic.tmpl b/tests/sample_data/read_only_templates/spiders/basic.tmpl deleted file mode 100644 index e9112bc95..000000000 --- a/tests/sample_data/read_only_templates/spiders/basic.tmpl +++ /dev/null @@ -1,10 +0,0 @@ -import scrapy - - -class $classname(scrapy.Spider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/'] - - def parse(self, response): - pass diff --git a/tests/sample_data/read_only_templates/spiders/crawl.tmpl b/tests/sample_data/read_only_templates/spiders/crawl.tmpl deleted file mode 100644 index 356496487..000000000 --- a/tests/sample_data/read_only_templates/spiders/crawl.tmpl +++ /dev/null @@ -1,20 +0,0 @@ -import scrapy -from scrapy.linkextractors import LinkExtractor -from scrapy.spiders import CrawlSpider, Rule - - -class $classname(CrawlSpider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/'] - - rules = ( - Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), - ) - - def parse_item(self, response): - item = {} - #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get() - #item['name'] = response.xpath('//div[@id="name"]').get() - #item['description'] = response.xpath('//div[@id="description"]').get() - return item diff --git a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl deleted file mode 100644 index cbcbe9e2c..000000000 --- a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl +++ /dev/null @@ -1,20 +0,0 @@ -from scrapy.spiders import CSVFeedSpider - - -class $classname(CSVFeedSpider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/feed.csv'] - # headers = ['id', 'name', 'description', 'image_link'] - # delimiter = '\t' - - # Do any adaptations you need here - #def adapt_response(self, response): - # return response - - def parse_row(self, response, row): - i = {} - #i['url'] = row['url'] - #i['name'] = row['name'] - #i['description'] = row['description'] - return i diff --git a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl deleted file mode 100644 index 5aa2aa8b0..000000000 --- a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -from scrapy.spiders import XMLFeedSpider - - -class $classname(XMLFeedSpider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/feed.xml'] - iterator = 'iternodes' # you can change this; see the docs - itertag = 'item' # change it accordingly - - def parse_node(self, response, selector): - item = {} - #item['url'] = selector.select('url').get() - #item['name'] = selector.select('name').get() - #item['description'] = selector.select('description').get() - return item diff --git a/tests/test_commands.py b/tests/test_commands.py index c25495d16..f3fe45139 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -2,6 +2,7 @@ import inspect import json import optparse import os +from stat import S_IWRITE as ANYONE_WRITE_PERMISSION import subprocess import sys import tempfile @@ -17,7 +18,7 @@ from twisted.trial import unittest import scrapy from scrapy.commands import ScrapyCommand -from scrapy.commands.startproject import IGNORE +from scrapy.commands.startproject import IGNORE, _make_writable from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv @@ -170,14 +171,14 @@ class StartprojectTemplatesTest(ProjectTest): template folder has the same permissions as in the project, i.e. everything is writable.""" scrapy_path = scrapy.__path__[0] - templates_dir = os.path.join(scrapy_path, 'templates', 'project') + project_template = os.path.join(scrapy_path, 'templates', 'project') project_name = 'startproject1' renamings = ( ('module', project_name), ('.tmpl', ''), ) expected_permissions = get_permissions_dict( - templates_dir, + project_template, renamings, IGNORE, ) @@ -209,22 +210,30 @@ class StartprojectTemplatesTest(ProjectTest): See https://github.com/scrapy/scrapy/pull/4604 """ scrapy_path = scrapy.__path__[0] - templates_dir = os.path.join(scrapy_path, 'templates', 'project') + templates_dir = os.path.join(scrapy_path, 'templates') + project_template = os.path.join(templates_dir, 'project') project_name = 'startproject2' renamings = ( ('module', project_name), ('.tmpl', ''), ) expected_permissions = get_permissions_dict( - templates_dir, + project_template, renamings, IGNORE, ) - tests_path = os.path.dirname(__file__) - read_only_templates_dir = os.path.join( - tests_path, 'sample_data', 'read_only_templates' - ) + def _make_read_only(path): + current_permissions = os.stat(path).st_mode + os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION) + + read_only_templates_dir = str(Path(mkdtemp()) / 'templates') + copytree(templates_dir, read_only_templates_dir) + + for root, dirs, files in os.walk(read_only_templates_dir): + for node in chain(dirs, files): + _make_read_only(os.path.join(root, node)) + destination = mkdtemp() process = subprocess.Popen( ( @@ -250,14 +259,14 @@ class StartprojectTemplatesTest(ProjectTest): """Check that pre-existing folders and files in the destination folder do not see their permissions modified.""" scrapy_path = scrapy.__path__[0] - templates_dir = os.path.join(scrapy_path, 'templates', 'project') + project_template = os.path.join(scrapy_path, 'templates', 'project') project_name = 'startproject3' renamings = ( ('module', project_name), ('.tmpl', ''), ) expected_permissions = get_permissions_dict( - templates_dir, + project_template, renamings, IGNORE, ) From 7e386157033e69bccc56e3a34d5545daa0174d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 7 Jul 2020 15:30:19 +0200 Subject: [PATCH 164/568] Remove unused import --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index f3fe45139..002237824 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -18,7 +18,7 @@ from twisted.trial import unittest import scrapy from scrapy.commands import ScrapyCommand -from scrapy.commands.startproject import IGNORE, _make_writable +from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv From 3e98ed24b6e9189d7fc7b4209d24971068274ddb Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Wed, 8 Jul 2020 17:13:57 +0530 Subject: [PATCH 165/568] Convert f-string to .format() --- tests/test_command_check.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 52005a4c5..72acd817c 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -14,20 +14,20 @@ class CheckCommandTest(CommandTest): def _write_contract(self, contracts, parse_def): with open(self.spider, 'w') as file: - file.write(f""" + file.write(""" import scrapy class CheckSpider(scrapy.Spider): - name = '{self.spider_name}' + name = '{0}' start_urls = ['http://example.com'] def parse(self, response, **cb_kwargs): \"\"\" @url http://www.amazon.com/s?field-keywords=selfish+gene - {contracts} + {1} \"\"\" - {parse_def} - """) + {2} + """.format(self.spider_name, contracts, parse_def)) def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) From 2ea7d82534cafe5b25b28ef5a78e5b714767d27a Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Wed, 8 Jul 2020 18:57:13 +0530 Subject: [PATCH 166/568] feat: H2ClientFactory --- scrapy/core/http2/protocol.py | 35 +++++++++++++++++++++-------- scrapy/core/http2/stream.py | 29 +++++++++++++----------- scrapy/core/http2/types.py | 13 ++++++----- tests/test_http2_client_protocol.py | 8 ++++--- 4 files changed, 55 insertions(+), 30 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 55dbcabec..ee51300cc 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,29 +2,38 @@ import ipaddress import itertools import logging from collections import deque -from typing import Dict, Optional +from typing import Dict, List, Optional from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - DataReceived, ResponseReceived, SettingsAcknowledged, + Event, DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) from h2.exceptions import ProtocolError from twisted.internet.defer import Deferred -from twisted.internet.protocol import connectionDone, Protocol +from twisted.internet.protocol import connectionDone, Factory, Protocol from twisted.internet.ssl import Certificate from twisted.python.failure import Failure +from twisted.web.client import URI from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request +from scrapy.settings import Settings logger = logging.getLogger(__name__) class H2ClientProtocol(Protocol): - def __init__(self) -> None: + def __init__(self, uri: URI, settings: Settings) -> None: + """ + Arguments: + uri -- URI of the base url to which HTTP/2 Connection will be made. + uri is used to verify that incoming client requests have correct + base URL. + settings -- Scrapy project settings + """ config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) @@ -53,8 +62,9 @@ class H2ClientProtocol(Protocol): self.metadata: H2ConnectionMetadataDict = { 'certificate': None, 'ip_address': None, - 'hostname': None, - 'port': None, + 'uri': uri, + 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'), + 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'), } @property @@ -135,8 +145,6 @@ class H2ClientProtocol(Protocol): destination = self.transport.getPeer() logger.debug('Connection made to {}'.format(destination)) self.metadata['ip_address'] = ipaddress.ip_address(destination.host) - self.metadata['port'] = destination.port - self.metadata['hostname'] = self.transport.transport.addr[0] self.conn.initiate_connection() self._write_to_transport() @@ -179,7 +187,7 @@ class H2ClientProtocol(Protocol): self._pending_request_stream_pool.clear() self.conn.close_connection() - def _handle_events(self, events: list) -> None: + def _handle_events(self, events: List[Event]) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler @@ -232,3 +240,12 @@ class H2ClientProtocol(Protocol): # Send leftover data for all the streams for stream in self.streams.values(): stream.receive_window_update() + + +class H2ClientFactory(Factory): + def __init__(self, uri: URI, settings: Settings) -> None: + self.uri = uri + self.settings = settings + + def buildProtocol(self, addr) -> H2ClientProtocol: + return H2ClientProtocol(self.uri, self.settings) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8b66d4b85..5017f9cd4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -16,6 +16,7 @@ from scrapy.core.http2.types import H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes +from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -34,7 +35,7 @@ class InactiveStreamClosed(ConnectionClosed): class InvalidHostname(Exception): - def __init__(self, request: Request, expected_hostname: Optional[str], expected_netloc: Optional[str]) -> None: + def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None: self.request = request self.expected_hostname = expected_hostname self.expected_netloc = expected_netloc @@ -83,10 +84,7 @@ class Stream: self, stream_id: int, request: Request, - protocol: "H2ClientProtocol", - download_maxsize: int = 0, - download_warnsize: int = 0, - fail_on_data_loss: bool = True + protocol: "H2ClientProtocol" ) -> None: """ Arguments: @@ -98,9 +96,14 @@ class Stream: self._request: Request = request self._protocol: "H2ClientProtocol" = protocol - self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) - self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) - self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss) + self._download_maxsize = self._request.meta.get( + 'download_maxsize', + self._protocol.metadata['default_download_maxsize'] + ) + self._download_warnsize = self._request.meta.get( + 'download_warnsize', + self._protocol.metadata['default_download_warnsize'] + ) self.request_start_time = None @@ -174,9 +177,9 @@ class Stream: # Make sure that we are sending the request to the correct URL url = urlparse(self._request.url) return ( - url.netloc == self._protocol.metadata['hostname'] - or url.netloc == f'{self._protocol.metadata["hostname"]}:{self._protocol.metadata["port"]}' - or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' + url.netloc == to_unicode(self._protocol.metadata['uri'].host) + or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc) + or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' ) def _get_request_headers(self) -> List[Tuple[str, str]]: @@ -391,8 +394,8 @@ class Stream: elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( self._request, - self._protocol.metadata['hostname'], - f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' + to_unicode(self._protocol.metadata['uri'].host), + f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' )) def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None: diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index dd7b1187b..d2aa1a9d8 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -3,6 +3,7 @@ from ipaddress import IPv4Address, IPv6Address from typing import Union, Optional from twisted.internet.ssl import Certificate +from twisted.web.client import URI # for python < 3.8 -- typing.TypedDict is undefined from typing_extensions import TypedDict @@ -19,14 +20,16 @@ class H2ConnectionMetadataDict(TypedDict): # is updated when HTTP/2 connection is made successfully ip_address: Optional[Union[IPv4Address, IPv6Address]] - # Name of the peer HTTP/2 connection is established - hostname: Optional[str] + # URI of the peer HTTP/2 connection is made + uri: URI - port: Optional[int] - - # Both ip_address and hostname are used by the Stream before + # Both ip_address and uri are used by the Stream before # initiating the request to verify that the base address + # Variables taken from Project Settings + default_download_maxsize: int + default_download_warnsize: int + class H2ResponseDict(TypedDict): # Data received frame by frame from the server is appended diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 9efca5267..05e5f5047 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -11,17 +11,18 @@ from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint -from twisted.internet.protocol import Factory from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase +from twisted.web.client import URI from twisted.web.http import Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File -from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.core.http2.protocol import H2ClientFactory from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest +from scrapy.settings import Settings from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -183,7 +184,8 @@ class Https2ClientProtocolTestCase(TestCase): trustRoot=self.client_certificate, acceptableProtocols=[b'h2'] ) - h2_client_factory = Factory.forProtocol(H2ClientProtocol) + uri = URI.fromBytes(to_bytes(self.get_url('/'))) + h2_client_factory = H2ClientFactory(uri, Settings()) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) From 75bff7b6d33bdc74c1a8eb0e43e4b484473c3062 Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Wed, 8 Jul 2020 19:48:42 +0530 Subject: [PATCH 167/568] Update url contract value --- tests/test_command_check.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 72acd817c..f27f526a3 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -23,7 +23,7 @@ class CheckSpider(scrapy.Spider): def parse(self, response, **cb_kwargs): \"\"\" - @url http://www.amazon.com/s?field-keywords=selfish+gene + @url http://example.com {1} \"\"\" {2} @@ -32,6 +32,7 @@ class CheckSpider(scrapy.Spider): def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) p, out, err = self.proc('check') + self.assertNotIn('F', out) self.assertIn('OK', err) self.assertEqual(p.returncode, 0) From 9e99be982a2916559219693e6665ec7e9319f0e9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Wed, 17 Jun 2020 15:52:57 -0300 Subject: [PATCH 168/568] Remove backslash --- scrapy/cmdline.py | 10 ++++++---- scrapy/commands/crawl.py | 6 ++++-- scrapy/commands/fetch.py | 6 ++++-- scrapy/commands/genspider.py | 7 ++++--- scrapy/commands/startproject.py | 7 ++++--- scrapy/commands/view.py | 3 +-- scrapy/core/engine.py | 8 +++++--- scrapy/downloadermiddlewares/redirect.py | 13 +++++++------ scrapy/downloadermiddlewares/retry.py | 6 ++++-- scrapy/extensions/memusage.py | 12 ++++++++---- scrapy/http/request/form.py | 3 +-- scrapy/http/response/text.py | 5 ++++- scrapy/link.py | 14 ++++++++++---- scrapy/settings/__init__.py | 3 +-- scrapy/utils/benchserver.py | 3 +-- scrapy/utils/conf.py | 12 +++++++----- scrapy/utils/curl.py | 3 +-- scrapy/utils/ossignal.py | 3 +-- scrapy/utils/response.py | 16 ++++++++++------ scrapy/utils/spider.py | 10 ++++++---- 20 files changed, 89 insertions(+), 61 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index b189e016b..3e88536e4 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -19,10 +19,12 @@ def _iter_command_classes(module_name): # scrapy.utils.spider.iter_spider_classes for module in walk_modules(module_name): for obj in vars(module).values(): - if inspect.isclass(obj) and \ - issubclass(obj, ScrapyCommand) and \ - obj.__module__ == module.__name__ and \ - not obj == ScrapyCommand: + if ( + inspect.isclass(obj) + and issubclass(obj, ScrapyCommand) + and obj.__module__ == module.__name__ + and not obj == ScrapyCommand + ): yield obj diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index e1724c1e6..f205c40b0 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -26,6 +26,8 @@ class Command(BaseRunSpiderCommand): else: self.crawler_process.start() - if self.crawler_process.bootstrap_failed or \ - (hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception): + if ( + self.crawler_process.bootstrap_failed + or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception + ): self.exitcode = 1 diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 063195f50..95f87e8c3 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -19,8 +19,10 @@ class Command(ScrapyCommand): return "Fetch a URL using the Scrapy downloader" def long_desc(self): - return "Fetch a URL using the Scrapy downloader and print its content " \ - "to stdout. You may want to use --nolog to disable logging" + return ( + "Fetch a URL using the Scrapy downloader and print its content" + " to stdout. You may want to use --nolog to disable logging" + ) def add_options(self, parser): ScrapyCommand.add_options(self, parser) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index abf3b7a5c..4c7548e9c 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -121,6 +121,7 @@ class Command(ScrapyCommand): @property def templates_dir(self): - _templates_base_dir = self.settings['TEMPLATES_DIR'] or \ - join(scrapy.__path__[0], 'templates') - return join(_templates_base_dir, 'spiders') + return join( + self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'), + 'spiders' + ) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 852281959..ae4a15b0f 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -137,6 +137,7 @@ class Command(ScrapyCommand): @property def templates_dir(self): - _templates_base_dir = self.settings['TEMPLATES_DIR'] or \ - join(scrapy.__path__[0], 'templates') - return join(_templates_base_dir, 'project') + return join( + self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'), + 'project' + ) diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 41e77ba3b..908bee966 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -8,8 +8,7 @@ class Command(fetch.Command): return "Open URL in browser, as seen by Scrapy" def long_desc(self): - return "Fetch a URL using the Scrapy downloader and show its " \ - "contents in a browser" + return "Fetch a URL using the Scrapy downloader and show its contents in a browser" def add_options(self, parser): super(Command, self).add_options(parser) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index de0da4b70..86a6abb23 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -141,10 +141,12 @@ class ExecutionEngine: def _needs_backout(self, spider): slot = self.slot - return not self.running \ - or slot.closing \ - or self.downloader.needs_backout() \ + return ( + not self.running + or slot.closing + or self.downloader.needs_backout() or self.scraper.slot.needs_backout() + ) def _next_request_from_scheduler(self, spider): slot = self.slot diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index b32afb8e4..366d60dcb 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -33,10 +33,8 @@ class BaseRedirectMiddleware: if ttl and redirects <= self.max_redirect_times: redirected.meta['redirect_times'] = redirects redirected.meta['redirect_ttl'] = ttl - 1 - redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \ - [request.url] - redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \ - [reason] + redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + [request.url] + redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + [reason] redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s", @@ -99,8 +97,11 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): self._maxdelay = settings.getint('METAREFRESH_MAXDELAY') def process_response(self, request, response, spider): - if request.meta.get('dont_redirect', False) or request.method == 'HEAD' or \ - not isinstance(response, HtmlResponse): + if ( + request.meta.get('dont_redirect', False) + or request.method == 'HEAD' + or not isinstance(response, HtmlResponse) + ): return response interval, url = get_meta_refresh(response, diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 6d11af5b2..67be8c282 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -60,8 +60,10 @@ class RetryMiddleware: return response def process_exception(self, request, exception, spider): - if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \ - and not request.meta.get('dont_retry', False): + if ( + isinstance(exception, self.EXCEPTIONS_TO_RETRY) + and not request.meta.get('dont_retry', False) + ): return self._retry(request, exception, spider) def _retry(self, request, reason, spider): diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index a0540bf8f..ab2e43e8c 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -81,8 +81,10 @@ class MemoryUsage: logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: - subj = "%s terminated: memory usage exceeded %dM at %s" % \ - (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + subj = ( + "%s terminated: memory usage exceeded %dM at %s" + % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) @@ -102,8 +104,10 @@ class MemoryUsage: logger.warning("Memory usage reached %(memusage)dM", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: - subj = "%s warning: memory usage reached %dM at %s" % \ - (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + subj = ( + "%s warning: memory usage reached %dM at %s" + % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/warning_notified', 1) self.warned = True diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index cd4e3373f..0e6ceef0b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -205,8 +205,7 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = u'.//*' + \ - u''.join(u'[@%s="%s"]' % c for c in clickdata.items()) + xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index b43fe5c19..0f300c8da 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -62,8 +62,11 @@ class TextResponse(Response): return self._declared_encoding() or self._body_inferred_encoding() def _declared_encoding(self): - return self._encoding or self._headers_encoding() \ + return ( + self._encoding + or self._headers_encoding() or self._body_declared_encoding() + ) def body_as_unicode(self): """Return body as unicode""" diff --git a/scrapy/link.py b/scrapy/link.py index 7cb0765cc..1ef50b113 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -21,12 +21,18 @@ class Link: self.nofollow = nofollow def __eq__(self, other): - return self.url == other.url and self.text == other.text and \ - self.fragment == other.fragment and self.nofollow == other.nofollow + return ( + self.url == other.url + and self.text == other.text + and self.fragment == other.fragment + and self.nofollow == other.nofollow + ) def __hash__(self): return hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow) def __repr__(self): - return 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' % \ - (self.url, self.text, self.fragment, self.nofollow) + return ( + 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' + % (self.url, self.text, self.fragment, self.nofollow) + ) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b9a13c018..ff8317cd1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -52,8 +52,7 @@ class SettingsAttribute: self.priority = priority def __str__(self): - return "<SettingsAttribute value={self.value!r} " \ - "priority={self.priority}>".format(self=self) + return "<SettingsAttribute value={self.value!r} priority={self.priority}>".format(self=self) __repr__ = __str__ diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 9d8d64612..f595a1acb 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -28,8 +28,7 @@ class Root(Resource): def _getarg(request, name, default=None, type=str): - return type(request.args[name][0]) \ - if name in request.args else default + return type(request.args[name][0]) if name in request.args else default if __name__ == '__main__': diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 5921f82bf..728bb5f1b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -101,11 +101,13 @@ def get_config(use_closest=True): def get_sources(use_closest=True): - xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \ - os.path.expanduser('~/.config') - sources = ['/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg', - xdg_config_home + '/scrapy.cfg', - os.path.expanduser('~/.scrapy.cfg')] + xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config') + sources = [ + '/etc/scrapy.cfg', + r'c:\scrapy\scrapy.cfg', + xdg_config_home + '/scrapy.cfg', + os.path.expanduser('~/.scrapy.cfg'), + ] if use_closest: sources.append(closest_scrapy_cfg()) return sources diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 67b22dbc5..aa681522f 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -9,8 +9,7 @@ from w3lib.http import basic_auth_header class CurlParser(argparse.ArgumentParser): def error(self, message): - error_msg = \ - 'There was an error parsing the curl command: {}'.format(message) + error_msg = 'There was an error parsing the curl command: {}'.format(message) raise ValueError(error_msg) diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 45c9cef0c..cf867f3f8 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -18,8 +18,7 @@ def install_shutdown_handlers(function, override_sigint=True): from twisted.internet import reactor reactor._handleSignals() signal.signal(signal.SIGTERM, function) - if signal.getsignal(signal.SIGINT) == signal.default_int_handler or \ - override_sigint: + if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows if hasattr(signal, 'SIGBREAK'): diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index edbc0db25..c29b619ce 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -47,13 +47,17 @@ def response_httprepr(response): is provided only for reference, since it's not the exact stream of bytes that was received (that's not exposed by Twisted). """ - s = b"HTTP/1.1 " + to_bytes(str(response.status)) + b" " + \ - to_bytes(http.RESPONSES.get(response.status, b'')) + b"\r\n" + values = [ + b"HTTP/1.1 ", + to_bytes(str(response.status)), + b" ", + to_bytes(http.RESPONSES.get(response.status, b'')), + b"\r\n", + ] if response.headers: - s += response.headers.to_string() + b"\r\n" - s += b"\r\n" - s += response.body - return s + values.extend([response.headers.to_string(), b"\r\n"]) + values.extend([b"\r\n", response.body]) + return b"".join(values) def open_in_browser(response, _openfunc=webbrowser.open): diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 7e7a50c88..f3a9a67a3 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -34,10 +34,12 @@ def iter_spider_classes(module): from scrapy.spiders import Spider for obj in vars(module).values(): - if inspect.isclass(obj) and \ - issubclass(obj, Spider) and \ - obj.__module__ == module.__name__ and \ - getattr(obj, 'name', None): + if ( + inspect.isclass(obj) + and issubclass(obj, Spider) + and obj.__module__ == module.__name__ + and getattr(obj, 'name', None) + ): yield obj From 9aea1f096171d38348b1403302c6c40eeef7f0a6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Thu, 9 Jul 2020 11:04:46 -0300 Subject: [PATCH 169/568] Remove backslash (tests) --- ...st_downloadermiddleware_httpcompression.py | 3 +- tests/test_downloadermiddleware_redirect.py | 18 ++++------- tests/test_selector.py | 3 +- tests/test_settings/__init__.py | 3 +- tests/test_spider.py | 10 ++++-- tests/test_spidermiddleware_referer.py | 32 +++++++++++++------ tests/test_utils_curl.py | 6 ++-- tests/test_utils_defer.py | 9 ++++-- tests/test_utils_iterators.py | 30 +++++++++-------- tests/test_utils_request.py | 8 +++-- tests/test_utils_response.py | 3 +- tests/test_utils_sitemap.py | 3 +- tests/test_utils_url.py | 3 +- tests/test_webclient.py | 5 ++- 14 files changed, 75 insertions(+), 61 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 87304d76c..a806f55ce 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -5,8 +5,7 @@ from gzip import GzipFile from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse -from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ - ACCEPTED_ENCODINGS +from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip from tests import tests_datadir diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index c46b1bb87..919dbed23 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -77,12 +77,9 @@ class RedirectMiddlewareTest(unittest.TestCase): assert isinstance(req2, Request) self.assertEqual(req2.url, url2) self.assertEqual(req2.method, 'GET') - assert 'Content-Type' not in req2.headers, \ - "Content-Type header must not be present in redirected request" - assert 'Content-Length' not in req2.headers, \ - "Content-Length header must not be present in redirected request" - assert not req2.body, \ - "Redirected body must be empty, not '%s'" % req2.body + assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" + assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" + assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body # response without Location header but with status code is 3XX should be ignored del rsp.headers['Location'] @@ -244,12 +241,9 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): assert isinstance(req2, Request) self.assertEqual(req2.url, 'http://example.org/newpage') self.assertEqual(req2.method, 'GET') - assert 'Content-Type' not in req2.headers, \ - "Content-Type header must not be present in redirected request" - assert 'Content-Length' not in req2.headers, \ - "Content-Length header must not be present in redirected request" - assert not req2.body, \ - "Redirected body must be empty, not '%s'" % req2.body + assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" + assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" + assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body def test_max_redirect_times(self): self.mw.max_redirect_times = 1 diff --git a/tests/test_selector.py b/tests/test_selector.py index bcf653444..00e663c11 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -88,8 +88,7 @@ class SelectorTestCase(unittest.TestCase): """Check that classes are using slots and are weak-referenceable""" x = Selector(text='') weakref.ref(x) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ - x.__class__.__name__ + assert not hasattr(x, '__dict__'), "%s does not use __slots__" % x.__class__.__name__ def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, 'received both response and text'): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 2da6aa4b5..6e56a28f5 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -86,8 +86,7 @@ class BaseSettingsTest(unittest.TestCase): def test_set_calls_settings_attributes_methods_on_update(self): attr = SettingsAttribute('value', 10) - with mock.patch.object(attr, '__setattr__') as mock_setattr, \ - mock.patch.object(attr, 'set') as mock_set: + with mock.patch.object(attr, '__setattr__') as mock_setattr, mock.patch.object(attr, 'set') as mock_set: self.settings.attributes = {'TEST_OPTION': attr} diff --git a/tests/test_spider.py b/tests/test_spider.py index 805d70459..bd9238810 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -11,8 +11,14 @@ from scrapy import signals from scrapy.settings import Settings from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse from scrapy.spiders.init import InitSpider -from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \ - CSVFeedSpider, SitemapSpider +from scrapy.spiders import ( + CSVFeedSpider, + CrawlSpider, + Rule, + SitemapSpider, + Spider, + XMLFeedSpider, +) from scrapy.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.test import get_crawler diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 067118cf0..5141f47af 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -6,16 +6,28 @@ from scrapy.http import Response, Request from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.downloadermiddlewares.redirect import RedirectMiddleware -from scrapy.spidermiddlewares.referer import RefererMiddleware, \ - POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ - POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ - POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \ - POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \ - DefaultReferrerPolicy, \ - NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ - OriginWhenCrossOriginPolicy, OriginPolicy, \ - StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \ - SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy +from scrapy.spidermiddlewares.referer import ( + DefaultReferrerPolicy, + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + OriginPolicy, + OriginWhenCrossOriginPolicy, + POLICY_NO_REFERRER, + POLICY_NO_REFERRER_WHEN_DOWNGRADE, + POLICY_ORIGIN, + POLICY_ORIGIN_WHEN_CROSS_ORIGIN, + POLICY_SAME_ORIGIN, + POLICY_SCRAPY_DEFAULT, + POLICY_STRICT_ORIGIN, + POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, + POLICY_UNSAFE_URL, + RefererMiddleware, + ReferrerPolicy, + SameOriginPolicy, + StrictOriginPolicy, + StrictOriginWhenCrossOriginPolicy, + UnsafeUrlPolicy, +) class TestRefererMiddleware(TestCase): diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 299a51efe..6b05c8771 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -29,8 +29,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): self._test_command(curl_command, expected_result) def test_get_basic_auth(self): - curl_command = 'curl "https://api.test.com/" -u ' \ - '"some_username:some_password"' + curl_command = 'curl "https://api.test.com/" -u "some_username:some_password"' expected_result = { "method": "GET", "url": "https://api.test.com/", @@ -212,8 +211,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): with warnings.catch_warnings(): # avoid warning when executing tests warnings.simplefilter('ignore') curl_command = 'curl --bar --baz http://www.example.com' - expected_result = \ - {"method": "GET", "url": "http://www.example.com"} + expected_result = {"method": "GET", "url": "http://www.example.com"} self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) # case 2: ignore_unknown_options=False (raise exception): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 2d4b88121..8c84331b9 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -2,8 +2,13 @@ from twisted.trial import unittest from twisted.internet import reactor, defer from twisted.python.failure import Failure -from scrapy.utils.defer import mustbe_deferred, process_chain, \ - process_chain_both, process_parallel, iter_errback +from scrapy.utils.defer import ( + iter_errback, + mustbe_deferred, + process_chain, + process_chain_both, + process_parallel, +) class MustbeDeferredTest(unittest.TestCase): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 8344c6701..3ebe3ac24 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -15,18 +15,20 @@ class XmliterTestCase(unittest.TestCase): xmliter = staticmethod(xmliter) def test_xmliter(self): - body = b"""<?xml version="1.0" encoding="UTF-8"?>\ + body = b""" + <?xml version="1.0" encoding="UTF-8"?> <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:noNamespaceSchemaLocation="someschmea.xsd">\ - <product id="001">\ - <type>Type 1</type>\ - <name>Name 1</name>\ - </product>\ - <product id="002">\ - <type>Type 2</type>\ - <name>Name 2</name>\ - </product>\ - </products>""" + xsi:noNamespaceSchemaLocation="someschmea.xsd"> + <product id="001"> + <type>Type 1</type> + <name>Name 1</name> + </product> + <product id="002"> + <type>Type 2</type> + <name>Name 2</name> + </product> + </products> + """ response = XmlResponse(url="http://example.com", body=body) attrs = [] @@ -115,7 +117,7 @@ class XmliterTestCase(unittest.TestCase): [[u'one'], [u'two']]) def test_xmliter_namespaces(self): - body = b"""\ + body = b""" <?xml version="1.0" encoding="UTF-8"?> <rss version="2.0" xmlns:g="http://base.google.com/ns/1.0"> <channel> @@ -185,7 +187,7 @@ class LxmlXmliterTestCase(XmliterTestCase): xmliter = staticmethod(xmliter_lxml) def test_xmliter_iterate_namespace(self): - body = b"""\ + body = b""" <?xml version="1.0" encoding="UTF-8"?> <rss version="2.0" xmlns="http://base.google.com/ns/1.0"> <channel> @@ -214,7 +216,7 @@ class LxmlXmliterTestCase(XmliterTestCase): self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg']) def test_xmliter_namespaces_prefix(self): - body = b"""\ + body = b""" <?xml version="1.0" encoding="UTF-8"?> <root> <h:table xmlns:h="http://www.w3.org/TR/html4/"> diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 4cd4b7010..7e0049b1d 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,7 +1,11 @@ import unittest from scrapy.http import Request -from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \ - request_authenticate, request_httprepr +from scrapy.utils.request import ( + _fingerprint_cache, + request_authenticate, + request_fingerprint, + request_httprepr, +) class UtilsRequestTest(unittest.TestCase): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 6ebf290c0..d6f4c0bb5 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -37,8 +37,7 @@ class ResponseUtilsTest(unittest.TestCase): self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody) return True response = HtmlResponse(url, body=body) - assert open_in_browser(response, _openfunc=browser_open), \ - "Browser not called" + assert open_in_browser(response, _openfunc=browser_open), "Browser not called" resp = Response(url, body=body) self.assertRaises(TypeError, open_in_browser, resp, debug=True) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index bfbf9abb3..23eb261b7 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -156,8 +156,7 @@ Disallow: /forum/active/ def test_sitemap_blanklines(self): """Assert we can deal with starting blank lines before <xml> tag""" - s = Sitemap(b"""\ - + s = Sitemap(b""" <?xml version="1.0" encoding="UTF-8"?> <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 09a6d6c70..a194a0998 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -207,8 +207,7 @@ def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) assert url.startswith(args[1]), \ - 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % ( - args[0], url, args[1]) + 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (args[0], url, args[1]) return do_expected diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 188e54602..c1c5945c2 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -356,9 +356,8 @@ class WebClientTestCase(unittest.TestCase): """ Test that non-standart body encoding matches Content-Encoding header """ body = b'\xd0\x81\xd1\x8e\xd0\xaf' - return getPage( - self.getURL('encoding'), body=body, response_transform=lambda r: r)\ - .addCallback(self._check_Encoding, body) + dfd = getPage(self.getURL('encoding'), body=body, response_transform=lambda r: r) + return dfd.addCallback(self._check_Encoding, body) def _check_Encoding(self, response, original_body): content_encoding = to_unicode(response.headers[b'Content-Encoding']) From cbe4dc57f3f65ecb851941dcfae0bc18c6c8582a Mon Sep 17 00:00:00 2001 From: Ajay Mittur <ajay.cs18@bmsce.ac.in> Date: Fri, 10 Jul 2020 18:22:43 +0530 Subject: [PATCH 170/568] Update pytest.ini --- pytest.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 92c5bcb75..ca8191f42 100644 --- a/pytest.ini +++ b/pytest.ini @@ -39,4 +39,5 @@ flake8-ignore = scrapy/utils/markup.py F403 scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 \ No newline at end of file + tests/test_loader.py E741 + From a6a5fa91da8944943e2c9d8f34f09662be17b781 Mon Sep 17 00:00:00 2001 From: Artur Shellunts <shellunts.artur@gmail.com> Date: Fri, 10 Jul 2020 23:10:49 +0200 Subject: [PATCH 171/568] Remove deprecated class HtmlParserLinkExtractor Issue #4356 --- scrapy/linkextractors/htmlparser.py | 91 ----------------------------- 1 file changed, 91 deletions(-) delete mode 100644 scrapy/linkextractors/htmlparser.py diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py deleted file mode 100644 index 0425d4340..000000000 --- a/scrapy/linkextractors/htmlparser.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -HTMLParser-based link extractor -""" -import warnings -from html.parser import HTMLParser -from urllib.parse import urljoin - -from w3lib.url import safe_url_string -from w3lib.html import strip_html5_whitespace - -from scrapy.link import Link -from scrapy.utils.python import unique as unique_list -from scrapy.exceptions import ScrapyDeprecationWarning - - -class HtmlParserLinkExtractor(HTMLParser): - - def __init__(self, tag="a", attr="href", process=None, unique=False, - strip=True): - HTMLParser.__init__(self) - - warnings.warn( - "HtmlParserLinkExtractor is deprecated and will be removed in " - "future releases. Please use scrapy.linkextractors.LinkExtractor", - ScrapyDeprecationWarning, stacklevel=2, - ) - - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.process_attr = process if callable(process) else lambda v: v - self.unique = unique - self.strip = strip - - def _extract_links(self, response_text, response_url, response_encoding): - self.reset() - self.feed(response_text) - self.close() - - links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links - - ret = [] - base_url = urljoin(response_url, self.base_url) if self.base_url else response_url - for link in links: - if isinstance(link.url, str): - link.url = link.url.encode(response_encoding) - try: - link.url = urljoin(base_url, link.url) - except ValueError: - continue - link.url = safe_url_string(link.url, response_encoding) - link.text = link.text.decode(response_encoding) - ret.append(link) - - return ret - - def extract_links(self, response): - # wrapper needed to allow to work directly with text - return self._extract_links(response.body, response.url, response.encoding) - - def reset(self): - HTMLParser.reset(self) - - self.base_url = None - self.current_link = None - self.links = [] - - def handle_starttag(self, tag, attrs): - if tag == 'base': - self.base_url = dict(attrs).get('href') - if self.scan_tag(tag): - for attr, value in attrs: - if self.scan_attr(attr): - if self.strip: - value = strip_html5_whitespace(value) - url = self.process_attr(value) - link = Link(url=url) - self.links.append(link) - self.current_link = link - - def handle_endtag(self, tag): - if self.scan_tag(tag): - self.current_link = None - - def handle_data(self, data): - if self.current_link: - self.current_link.text = self.current_link.text + data - - def matches(self, url): - """This extractor matches with any url, since - it doesn't contain any patterns""" - return True From 3f7e8635f479f4de2ab1c3d518010730a5f6f0a6 Mon Sep 17 00:00:00 2001 From: Aditya Kumar <k.aditya00@gmail.com> Date: Sat, 11 Jul 2020 12:18:24 +0530 Subject: [PATCH 172/568] Allow the parse command to write data to a file (#4377) --- docs/topics/commands.rst | 4 +++- scrapy/commands/__init__.py | 2 +- scrapy/commands/parse.py | 24 ++++++++---------------- tests/test_command_parse.py | 23 ++++++++++++++++++++++- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index a0dcba90d..4fce51abc 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -468,7 +468,7 @@ Supported options: * ``--callback`` or ``-c``: spider method to use as callback for parsing the response -* ``--meta`` or ``-m``: additional request meta that will be passed to the callback +* ``--meta`` or ``-m``: additional request meta that will be passed to the callback request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' * ``--cbkwargs``: additional keyword arguments that will be passed to the callback. @@ -491,6 +491,8 @@ Supported options: * ``--verbose`` or ``-v``: display information for each depth level +* ``--output`` or ``-o``: dump scraped items to a file + .. skip: start Usage example:: diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index ab850dcb3..57ce4e522 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -108,7 +108,7 @@ class ScrapyCommand: class BaseRunSpiderCommand(ScrapyCommand): """ - Common class used to share functionality between the crawl and runspider commands + Common class used to share functionality between the crawl, parse and runspider commands """ def add_options(self, parser): ScrapyCommand.add_options(self, parser) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 8b7fa8b58..abc8ba9ff 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -4,18 +4,16 @@ import logging from itemadapter import is_item, ItemAdapter from w3lib.url import is_url -from scrapy.commands import ScrapyCommand +from scrapy.commands import BaseRunSpiderCommand from scrapy.http import Request from scrapy.utils import display -from scrapy.utils.conf import arglist_to_dict from scrapy.utils.spider import iterate_spider_output, spidercls_for_request from scrapy.exceptions import UsageError logger = logging.getLogger(__name__) -class Command(ScrapyCommand): - +class Command(BaseRunSpiderCommand): requires_project = True spider = None @@ -31,11 +29,9 @@ class Command(ScrapyCommand): return "Parse URL (using its spider) and print the results" def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + BaseRunSpiderCommand.add_options(self, parser) parser.add_option("--spider", dest="spider", default=None, help="use this spider without looking for one") - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") parser.add_option("--pipelines", action="store_true", help="process items through pipelines") parser.add_option("--nolinks", dest="nolinks", action="store_true", @@ -200,12 +196,15 @@ class Command(ScrapyCommand): self.add_items(depth, items) self.add_requests(depth, requests) + scraped_data = items if opts.output else [] if depth < opts.depth: for req in requests: req.meta['_depth'] = depth + 1 req.meta['_callback'] = req.callback req.callback = callback - return requests + scraped_data += requests + + return scraped_data # update request meta if any extra meta was passed through the --meta/-m opts. if opts.meta: @@ -221,18 +220,11 @@ class Command(ScrapyCommand): return request def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) + BaseRunSpiderCommand.process_options(self, args, opts) - self.process_spider_arguments(opts) self.process_request_meta(opts) self.process_request_cb_kwargs(opts) - def process_spider_arguments(self, opts): - try: - opts.spargs = arglist_to_dict(opts.spargs) - except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - def process_request_meta(self, opts): if opts.meta: try: diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index a09dcf072..5754a5478 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,5 +1,5 @@ import os -from os.path import join, abspath +from os.path import join, abspath, isfile, exists from twisted.internet import defer from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest @@ -218,3 +218,24 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} ) self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) + + @defer.inlineCallbacks + def test_output_flag(self): + """Checks if a file was created successfully having + correct format containing correct data in it. + """ + file_name = 'data.json' + file_path = join(self.proj_path, file_name) + yield self.execute([ + '--spider', self.spider_name, + '-c', 'parse', + '-o', file_name, + self.url('/html') + ]) + + self.assertTrue(exists(file_path)) + self.assertTrue(isfile(file_path)) + + content = '[\n{},\n{"foo": "bar"}\n]' + with open(file_path, 'r') as f: + self.assertEqual(f.read(), content) From 64c6af10e1b276db68ea722845621912d2023a75 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Mon, 13 Jul 2020 00:57:49 +0530 Subject: [PATCH 173/568] refactor: use str instead of to_unicode --- scrapy/core/http2/stream.py | 7 +++---- tests/test_http2_client_protocol.py | 29 ++++++++++++++++------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 5017f9cd4..a2f0e2aa4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -16,7 +16,6 @@ from scrapy.core.http2.types import H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes -from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -177,8 +176,8 @@ class Stream: # Make sure that we are sending the request to the correct URL url = urlparse(self._request.url) return ( - url.netloc == to_unicode(self._protocol.metadata['uri'].host) - or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc) + url.netloc == str(self._protocol.metadata['uri'].host, 'utf-8') + or url.netloc == str(self._protocol.metadata['uri'].netloc, 'utf-8') or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' ) @@ -394,7 +393,7 @@ class Stream: elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( self._request, - to_unicode(self._protocol.metadata['uri'].host), + str(self._protocol.metadata['uri'].host, 'utf-8'), f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' )) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 05e5f5047..05f4889ef 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -23,7 +23,6 @@ from scrapy.core.http2.protocol import H2ClientFactory from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings -from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -39,7 +38,7 @@ def make_html_body(val): <h1>Hello from HTTP2<h1> <p>{val}</p> </html>''' - return to_bytes(response) + return bytes(response, 'utf-8') class Data: @@ -83,10 +82,11 @@ class PostDataJsonMixin: 'extra-data': extra_data } for k, v in request.requestHeaders.getAllRawHeaders(): - response['request-headers'][to_unicode(k)] = to_unicode(v[0]) + response['request-headers'][str(k, 'utf-8')] = str(v[0], 'utf-8') - response_bytes = to_bytes(json.dumps(response)) - request.setHeader('Content-Type', 'application/json') + response_bytes = bytes(json.dumps(response), 'utf-8') + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') return response_bytes @@ -127,13 +127,14 @@ class NoContentLengthHeader(LeafResource): class QueryParams(LeafResource): def render_GET(self, request: TxRequest): - request.setHeader('Content-Type', 'application/json') + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') query_params = {} for k, v in request.args.items(): - query_params[to_unicode(k)] = to_unicode(v[0]) + query_params[str(k, 'utf-8')] = str(v[0], 'utf-8') - return to_bytes(json.dumps(query_params)) + return bytes(json.dumps(query_params), 'utf-8') def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: @@ -184,7 +185,7 @@ class Https2ClientProtocolTestCase(TestCase): trustRoot=self.client_certificate, acceptableProtocols=[b'h2'] ) - uri = URI.fromBytes(to_bytes(self.get_url('/'))) + uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) h2_client_factory = H2ClientFactory(uri, Settings()) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) @@ -278,7 +279,8 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(len(response.body), content_length) # Parse the body - body = json.loads(to_unicode(response.body)) + content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + body = json.loads(str(response.body, content_encoding)) self.assertIn('request-body', body) self.assertIn('extra-data', body) self.assertIn('request-headers', body) @@ -292,9 +294,9 @@ class Https2ClientProtocolTestCase(TestCase): # Check if headers were sent successfully request_headers = body['request-headers'] for k, v in request.headers.items(): - k_str = to_unicode(k) + k_str = str(k, 'utf-8') self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], to_unicode(v[0])) + self.assertEqual(request_headers[k_str], str(v[0], 'utf-8')) d.addCallback(assert_response) d.addErrback(self.fail) @@ -494,7 +496,8 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(self.get_url(f'/query-params?{urlencode(params)}')) def assert_query_params(response: Response): - data = json.loads(to_unicode(response.body)) + content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + data = json.loads(str(response.body, content_encoding)) self.assertEqual(data, params) d = self.client.request(request) From d54c4496ee57785f3d6f882e2d128bb64b6b262c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 13 Jul 2020 14:36:33 +0200 Subject: [PATCH 174/568] Refactor guess_scheme --- scrapy/utils/url.py | 60 ++++++++++++++++++++++++++++++----------- tests/test_utils_url.py | 33 +++++++++++++++++++++-- 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 60e2b50eb..1e431d3bf 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -83,26 +83,54 @@ def add_http_if_no_scheme(url): return url +def _is_posix_path(string): + return bool( + re.match( + r''' + ^ # start with... + ( + \. # ...a single dot, + ( + \. | [^/\.]+ # optionally followed by + )? # either a second dot or some characters + | + ~ # $HOME + )? # optional match of ".", ".." or ".blabla" + / # at least one "/" for a file path, + . # and something after the "/" + ''', + string, + flags=re.VERBOSE, + ) + ) + + +def _is_windows_path(string): + return bool( + re.match( + r''' + ^ + ( + [a-z]:\\ + | \\\\ + ) + ''', + string, + flags=re.IGNORECASE | re.VERBOSE, + ) + ) + + +def _is_path(string): + return _is_posix_path(string) or _is_windows_path(string) + + def guess_scheme(url): """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" - # POSIX path - if re.match(r'''^ # start with... - ( - \. # ...a single dot, - ( - \. | [^/\.]+ # optionally followed by - )? # either a second dot or some characters - )? # optional match of ".", ".." or ".blabla" - / # at least one "/" for a file path, - . # and something after the "/" - ''', url, flags=re.VERBOSE): + if _is_path(url): return any_to_uri(url) - # Windows drive-letter path - elif re.match(r'''^[a-z]:\\''', url, flags=re.IGNORECASE): - return any_to_uri(url) - else: - return add_http_if_no_scheme(url) + return add_http_if_no_scheme(url) def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 09a6d6c70..6a5254d54 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,8 +1,14 @@ import unittest from scrapy.spiders import Spider -from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - add_http_if_no_scheme, guess_scheme, strip_url) +from scrapy.utils.url import ( + add_http_if_no_scheme, + guess_scheme, + _is_path, + strip_url, + url_is_from_any_domain, + url_is_from_spider, +) __doctests__ = ['scrapy.utils.url'] @@ -434,5 +440,28 @@ class StripUrl(unittest.TestCase): self.assertEqual(strip_url(i, origin_only=True), o) +class IsPathTestCase(unittest.TestCase): + + def test_path(self): + for input_value, output_value in ( + # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell + # Unix-like OS, Microsoft Windows / cmd.exe + ("/home/user/docs/Letter.txt", True), + ("./inthisdir", True), + ("../../greatgrandparent", True), + ("~/.rcinfo", True), + (r"C:\user\docs\Letter.txt", True), + ("/user/docs/Letter.txt", True), + (r"C:\Letter.txt", True), + (r"\\Server01\user\docs\Letter.txt", True), + (r"\\?\UNC\Server01\user\docs\Letter.txt", True), + (r"\\?\C:\user\docs\Letter.txt", True), + (r"C:\user\docs\somefile.ext:alternate_stream_name", True), + + (r"https://example.com", False), + ): + self.assertEqual(_is_path(input_value), output_value, input_value) + + if __name__ == "__main__": unittest.main() From 53c323b19d81784e6c376ce8b9602de24d8e3037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 13 Jul 2020 15:29:30 +0200 Subject: [PATCH 175/568] =?UTF-8?q?=5Fis=5Fpath=20=E2=86=92=20=5Fis=5Ffile?= =?UTF-8?q?system=5Fpath?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/utils/url.py | 4 ++-- tests/test_utils_url.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 1e431d3bf..b23ddb459 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -121,14 +121,14 @@ def _is_windows_path(string): ) -def _is_path(string): +def _is_filesystem_path(string): return _is_posix_path(string) or _is_windows_path(string) def guess_scheme(url): """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" - if _is_path(url): + if _is_filesystem_path(url): return any_to_uri(url) return add_http_if_no_scheme(url) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 6a5254d54..3a143ba2f 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -4,7 +4,7 @@ from scrapy.spiders import Spider from scrapy.utils.url import ( add_http_if_no_scheme, guess_scheme, - _is_path, + _is_filesystem_path, strip_url, url_is_from_any_domain, url_is_from_spider, @@ -460,7 +460,7 @@ class IsPathTestCase(unittest.TestCase): (r"https://example.com", False), ): - self.assertEqual(_is_path(input_value), output_value, input_value) + self.assertEqual(_is_filesystem_path(input_value), output_value, input_value) if __name__ == "__main__": From aeaeb7385b7d1e6570bad38da4db492de8fb4206 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Tue, 14 Jul 2020 03:55:14 +0530 Subject: [PATCH 176/568] feat: assert negotiated protocol as h2 - implement IHandshakeListener in H2ClientProtocol to know when handshake is completed - implement IProtocolNegotiationFactory in H2ClientFactory to provide information about the acceptableProtols (h2) during NPN or ALPN protocol --- scrapy/core/http2/protocol.py | 69 ++++++++++++++++++++++------- scrapy/core/http2/stream.py | 4 +- tests/test_http2_client_protocol.py | 2 +- 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index ee51300cc..5d859d475 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -12,10 +12,12 @@ from h2.events import ( ) from h2.exceptions import ProtocolError from twisted.internet.defer import Deferred +from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory from twisted.internet.protocol import connectionDone, Factory, Protocol from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from twisted.web.client import URI +from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict @@ -25,15 +27,29 @@ from scrapy.settings import Settings logger = logging.getLogger(__name__) +class InvalidNegotiatedProtocol(ProtocolError): + + def __init__(self, negotiated_protocol: str) -> None: + self.negotiated_protocol = negotiated_protocol + + def __str__(self) -> str: + return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}' + + +@implementer(IHandshakeListener) class H2ClientProtocol(Protocol): - def __init__(self, uri: URI, settings: Settings) -> None: + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: """ Arguments: uri -- URI of the base url to which HTTP/2 Connection will be made. uri is used to verify that incoming client requests have correct base URL. settings -- Scrapy project settings + conn_lost_deferred -- Deferred fires with the reason: Failure to notify + that connection was lost """ + self._conn_lost_deferred = conn_lost_deferred + config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) @@ -55,9 +71,9 @@ class H2ClientProtocol(Protocol): # We use simple FIFO policy to handle pending requests self._active_streams = 0 - # Save an instance of ProtocolError raised by hyper-h2 - # We pass this instance to the streams ResponseFailed() failure - self._protocol_error: Optional[ProtocolError] = None + # Save an instance of errors raised which lead to losing the connection + # We pass these instances to the streams ResponseFailed() failure + self._conn_lost_errors: List[BaseException] = [] self.metadata: H2ConnectionMetadataDict = { 'certificate': None, @@ -136,6 +152,12 @@ class H2ClientProtocol(Protocol): # Add the stream to the request pool self._pending_request_stream_pool.append(stream) + + # If we are connection and receive a request + # There is a good chance that the connection was IDLE + # Hence, we need to initiate pending requests + if self.is_connected: + self._send_pending_requests() return d def connectionMade(self) -> None: @@ -149,6 +171,19 @@ class H2ClientProtocol(Protocol): self.conn.initiate_connection() self._write_to_transport() + def _lose_connection_with_error(self, errors: List[BaseException]): + """Helper function to lose the connection with the error sent as a + reason""" + self._conn_lost_errors += errors + self.transport.loseConnection() + + def handshakeCompleted(self): + """We close the connection with InvalidNegotiatedProtocol exception + when the connection was not made via h2 protocol""" + negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') + if negotiated_protocol != 'h2': + self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) + def dataReceived(self, data: bytes) -> None: try: events = self.conn.receive_data(data) @@ -157,10 +192,7 @@ class H2ClientProtocol(Protocol): # Save this error as ultimately the connection will be dropped # internally by hyper-h2. Saved error will be passed to all the streams # closed with the connection. - self._protocol_error = e - - # We lose the transport connection here - self.transport.loseConnection() + self._lose_connection_with_error([e]) finally: self._write_to_transport() @@ -168,17 +200,17 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ - errors = [] + # Notify the connection pool instance such that no new requests are + # sent over current connection if not reason.check(connectionDone): - logger.warning("Connection lost with reason " + str(reason)) - errors.append(reason) + self._conn_lost_errors.append(reason) - if self._protocol_error: - errors.append(self._protocol_error) + if self._conn_lost_deferred: + self._conn_lost_deferred.callback(self._conn_lost_errors) for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, errors, from_protocol=True) + stream.close(StreamCloseReason.CONNECTION_LOST, self._conn_lost_errors, from_protocol=True) else: stream.close(StreamCloseReason.INACTIVE, from_protocol=True) @@ -242,10 +274,15 @@ class H2ClientProtocol(Protocol): stream.receive_window_update() +@implementer(IProtocolNegotiationFactory) class H2ClientFactory(Factory): - def __init__(self, uri: URI, settings: Settings) -> None: + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: self.uri = uri self.settings = settings + self.conn_lost_deferred = conn_lost_deferred def buildProtocol(self, addr) -> H2ClientProtocol: - return H2ClientProtocol(self.uri, self.settings) + return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) + + def acceptableProtocols(self) -> List[bytes]: + return [b'h2'] diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a2f0e2aa4..16d54b2fc 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -44,7 +44,7 @@ class InvalidHostname(Exception): class StreamCloseReason(Enum): - # Received a StreamEnded event + # Received a StreamEnded event from the remote ENDED = 1 # Received a StreamReset event -- ended abruptly @@ -329,7 +329,7 @@ class Stream: def close( self, reason: StreamCloseReason, - errors: Optional[List[Exception]] = None, + errors: Optional[List[BaseException]] = None, from_protocol: bool = False ) -> None: """Based on the reason sent we will handle each case. diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 05f4889ef..7fcea58c5 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -9,7 +9,7 @@ from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError +from twisted.internet.defer import CancelledError, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure From ed5247ca4cbef98d1acf499c888ded74e444ef48 Mon Sep 17 00:00:00 2001 From: Artur Shellunts <shellunts.artur@gmail.com> Date: Tue, 14 Jul 2020 18:06:11 +0200 Subject: [PATCH 177/568] Remove htmlparser.py from tests/ignore.txt --- tests/ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ignores.txt b/tests/ignores.txt index 45cf6fb92..f6e0d6fbe 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -1,6 +1,5 @@ scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py -scrapy/linkextractors/htmlparser.py scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py scrapy/extensions/memusage.py From 1dd27a92fa53be87c71df43bd6f3043a225a2c10 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Tue, 14 Jul 2020 17:58:22 +0530 Subject: [PATCH 178/568] feat: Idle Timeout for H2Connection (240s) --- scrapy/core/http2/protocol.py | 60 ++++++++++++++++++++++++----- scrapy/core/http2/stream.py | 3 +- tests/test_http2_client_protocol.py | 10 +++-- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 5d859d475..bf41a2805 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -6,15 +6,18 @@ from typing import Dict, List, Optional from h2.config import H2Configuration from h2.connection import H2Connection +from h2.errors import ErrorCodes from h2.events import ( Event, DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) from h2.exceptions import ProtocolError from twisted.internet.defer import Deferred +from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory from twisted.internet.protocol import connectionDone, Factory, Protocol from twisted.internet.ssl import Certificate +from twisted.protocols.policies import TimeoutMixin from twisted.python.failure import Failure from twisted.web.client import URI from zope.interface import implementer @@ -37,7 +40,9 @@ class InvalidNegotiatedProtocol(ProtocolError): @implementer(IHandshakeListener) -class H2ClientProtocol(Protocol): +class H2ClientProtocol(Protocol, TimeoutMixin): + IDLE_TIMEOUT = 240 + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: """ Arguments: @@ -71,6 +76,10 @@ class H2ClientProtocol(Protocol): # We use simple FIFO policy to handle pending requests self._active_streams = 0 + # Flag to keep track if settings were acknowledged by the remote + # This ensures that we have established a HTTP/2 connection + self._settings_acknowledged = False + # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure self._conn_lost_errors: List[BaseException] = [] @@ -84,12 +93,12 @@ class H2ClientProtocol(Protocol): } @property - def is_connected(self) -> bool: + def h2_connected(self) -> bool: """Boolean to keep track of the connection status. This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection """ - return bool(self.transport.connected) + return bool(self.transport.connected) and self._settings_acknowledged @property def allowed_max_concurrent_streams(self) -> int: @@ -111,7 +120,7 @@ class H2ClientProtocol(Protocol): while ( self._pending_request_stream_pool and self._active_streams < self.allowed_max_concurrent_streams - and self.is_connected + and self.h2_connected ): self._active_streams += 1 stream = self._pending_request_stream_pool.popleft() @@ -140,6 +149,9 @@ class H2ClientProtocol(Protocol): """ Write data to the underlying transport connection from the HTTP2 connection instance if any """ + # Reset the idle timeout as connection is still actively sending data + self.resetTimeout() + data = self.conn.data_to_send() self.transport.write(data) @@ -153,21 +165,23 @@ class H2ClientProtocol(Protocol): # Add the stream to the request pool self._pending_request_stream_pool.append(stream) - # If we are connection and receive a request - # There is a good chance that the connection was IDLE - # Hence, we need to initiate pending requests - if self.is_connected: - self._send_pending_requests() + # If we receive a request when connection is idle + # We need to initiate pending requests + self._send_pending_requests() return d def connectionMade(self) -> None: """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ + # Initialize the timeout + self.setTimeout(self.IDLE_TIMEOUT) + destination = self.transport.getPeer() logger.debug('Connection made to {}'.format(destination)) self.metadata['ip_address'] = ipaddress.ip_address(destination.host) + # Initiate H2 Connection self.conn.initiate_connection() self._write_to_transport() @@ -185,6 +199,9 @@ class H2ClientProtocol(Protocol): self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) def dataReceived(self, data: bytes) -> None: + # Reset the idle timeout as connection is still actively receiving data + self.resetTimeout() + try: events = self.conn.receive_data(data) self._handle_events(events) @@ -196,10 +213,33 @@ class H2ClientProtocol(Protocol): finally: self._write_to_transport() + def timeoutConnection(self): + """Called when the connection times out. + We lose the connection with TimeoutError""" + + # Check whether there are open streams. If there are, we're going to + # want to use the error code PROTOCOL_ERROR. If there aren't, use + # NO_ERROR. + if ( + self.conn.open_outbound_streams > 0 + or self.conn.open_inbound_streams > 0 + or self._active_streams > 0 + ): + error_code = ErrorCodes.PROTOCOL_ERROR + else: + error_code = ErrorCodes.NO_ERROR + self.conn.close_connection(error_code=error_code) + self._write_to_transport() + + self._lose_connection_with_error([TimeoutError("Hello")]) + def connectionLost(self, reason: Failure = connectionDone) -> None: """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ + # Cancel the timeout if not done yet + self.setTimeout(None) + # Notify the connection pool instance such that no new requests are # sent over current connection if not reason.check(connectionDone): @@ -250,6 +290,8 @@ class H2ClientProtocol(Protocol): self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: + self._settings_acknowledged = True + # Send off all the pending requests as now we have # established a proper HTTP/2 connection self._send_pending_requests() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 16d54b2fc..f60691945 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -134,7 +134,8 @@ class Stream: def _cancel(_): # Close this stream as gracefully as possible - # Check if the stream has started + # If the associated request is initiated we reset this stream + # else we directly call close() method if self.request_sent: self.reset_stream(StreamCloseReason.CANCELLED) else: diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7fcea58c5..2833801e7 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -9,7 +9,7 @@ from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor -from twisted.internet.defer import CancelledError, DeferredList, inlineCallbacks +from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure @@ -174,6 +174,7 @@ class Https2ClientProtocolTestCase(TestCase): # Start server for testing self.hostname = u'localhost' context_factory = ssl_context_factory(self.key_file, self.certificate_file) + server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) self.server = yield server_endpoint.listen(self.site) self.port_number = self.server.getHost().port @@ -186,17 +187,20 @@ class Https2ClientProtocolTestCase(TestCase): acceptableProtocols=[b'h2'] ) uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) - h2_client_factory = H2ClientFactory(uri, Settings()) + + self.conn_closed_deferred = Deferred() + h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) @inlineCallbacks def tearDown(self): - if self.client.is_connected: + if self.client.connected: yield self.client.transport.loseConnection() yield self.client.transport.abortConnection() yield self.server.stopListening() shutil.rmtree(self.temp_directory) + self.conn_closed_deferred = None def get_url(self, path): """ From e662762e6ab2f53ff2e31e21f34c078590f65aa9 Mon Sep 17 00:00:00 2001 From: Aditya <k.aditya00@gmail.com> Date: Wed, 15 Jul 2020 03:45:32 +0530 Subject: [PATCH 179/568] chore: Handle ConnectionTerminated event --- scrapy/core/http2/protocol.py | 44 +++++++++++++++++++++++++++-------- scrapy/core/http2/stream.py | 4 ++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index bf41a2805..041908116 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,16 +2,18 @@ import ipaddress import itertools import logging from collections import deque -from typing import Dict, List, Optional +from ipaddress import IPv4Address, IPv6Address +from typing import Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection from h2.errors import ErrorCodes from h2.events import ( - Event, DataReceived, ResponseReceived, SettingsAcknowledged, - StreamEnded, StreamReset, WindowUpdated + Event, ConnectionTerminated, DataReceived, ResponseReceived, + SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived, + WindowUpdated ) -from h2.exceptions import ProtocolError +from h2.exceptions import H2Error, ProtocolError from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory @@ -30,7 +32,7 @@ from scrapy.settings import Settings logger = logging.getLogger(__name__) -class InvalidNegotiatedProtocol(ProtocolError): +class InvalidNegotiatedProtocol(H2Error): def __init__(self, negotiated_protocol: str) -> None: self.negotiated_protocol = negotiated_protocol @@ -39,6 +41,15 @@ class InvalidNegotiatedProtocol(ProtocolError): return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}' +class RemoteTerminatedConnection(H2Error): + def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], event: ConnectionTerminated): + self.remote_ip_address = remote_ip_address + self.terminate_event = event + + def __str__(self) -> str: + return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address}' + + @implementer(IHandshakeListener) class H2ClientProtocol(Protocol, TimeoutMixin): IDLE_TIMEOUT = 240 @@ -194,8 +205,12 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def handshakeCompleted(self): """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" - negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') + negotiated_protocol = self.transport.negotiatedProtocol + if type(negotiated_protocol) is bytes: + negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') if negotiated_protocol != 'h2': + # Here we have not initiated the connection yet + # So, no need to send a GOAWAY frame to the remote self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) def dataReceived(self, data: bytes) -> None: @@ -231,7 +246,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.conn.close_connection(error_code=error_code) self._write_to_transport() - self._lose_connection_with_error([TimeoutError("Hello")]) + self._lose_connection_with_error([ + TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s") + ]) def connectionLost(self, reason: Failure = connectionDone) -> None: """Called by Twisted when the transport connection is lost. @@ -267,7 +284,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin): events -- A list of events that the remote peer triggered by sending data """ for event in events: - if isinstance(event, DataReceived): + if isinstance(event, ConnectionTerminated): + self.connection_terminated(event) + elif isinstance(event, DataReceived): self.data_received(event) elif isinstance(event, ResponseReceived): self.response_received(event) @@ -279,10 +298,15 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.window_updated(event) elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) - else: - logger.debug('Received unhandled event {}'.format(event)) + elif isinstance(event, UnknownFrameReceived): + logger.debug(f'UnknownFrameReceived: frame={event.frame}') # Event handler functions starts here + def connection_terminated(self, event: ConnectionTerminated) -> None: + self._lose_connection_with_error([ + RemoteTerminatedConnection(self.metadata['ip_address'], event) + ]) + def data_received(self, event: DataReceived) -> None: self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index f60691945..15f081cab 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -5,7 +5,7 @@ from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes -from h2.exceptions import StreamClosedError +from h2.exceptions import H2Error, StreamClosedError from hpack import HeaderTuple from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed @@ -32,7 +32,7 @@ class InactiveStreamClosed(ConnectionClosed): self.request = request -class InvalidHostname(Exception): +class InvalidHostname(H2Error): def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None: self.request = request From 38496a00b7d2bcfa9d435551409e2c44007d168d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com> Date: Wed, 15 Jul 2020 07:08:36 -0300 Subject: [PATCH 180/568] Use the itemlaoders library (#4516) --- docs/conf.py | 13 + docs/topics/loaders.rst | 418 +----------------- scrapy/loader/__init__.py | 269 +++--------- scrapy/loader/common.py | 19 +- scrapy/loader/processors.py | 99 +---- scrapy/utils/misc.py | 6 + setup.cfg | 3 + setup.py | 1 + tests/requirements-py3.txt | 1 - tests/test_loader.py | 531 +---------------------- tests/test_loader_deprecated.py | 720 ++++++++++++++++++++++++++++++++ 11 files changed, 854 insertions(+), 1226 deletions(-) create mode 100644 tests/test_loader_deprecated.py diff --git a/docs/conf.py b/docs/conf.py index 86734fae7..427c79481 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -284,6 +284,7 @@ intersphinx_mapping = { 'attrs': ('https://www.attrs.org/en/stable/', None), 'coverage': ('https://coverage.readthedocs.io/en/stable', None), 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), + 'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None), 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), @@ -305,3 +306,15 @@ hoverxref_role_types = { "ref": "tooltip", } hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal'] + + +def setup(app): + app.connect('autodoc-skip-member', maybe_skip_member) + + +def maybe_skip_member(app, what, name, obj, skip, options): + if not skip: + # autodocs was generating a text "alias of" for the following members + # https://github.com/sphinx-doc/sphinx/issues/4422 + return name in {'default_item_class', 'default_selector_class'} + return skip diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 9c82bb4d9..d0eeb4097 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -20,6 +20,10 @@ Item Loaders are designed to provide a flexible, efficient and easy mechanism for extending and overriding different field parsing rules, either by spider, or by source format (HTML, XML, etc) without becoming a nightmare to maintain. +.. note:: Item Loaders are an extension of the itemloaders_ library that make it + easier to work with Scrapy by adding support for + :ref:`responses <topics-request-response>`. + Using Item Loaders to populate items ==================================== @@ -173,8 +177,8 @@ The other thing you need to keep in mind is that the values returned by input processors are collected internally (in lists) and then passed to output processors to populate the fields. -Last, but not least, Scrapy comes with some :ref:`commonly used processors -<topics-loaders-available-processors>` built-in for convenience. +Last, but not least, itemloaders_ comes with some :ref:`commonly used +processors <itemloaders:built-in-processors>` built-in for convenience. Declaring Item Loaders @@ -182,8 +186,8 @@ Declaring Item Loaders Item Loaders are declared using a class definition syntax. Here is an example:: + from itemloaders.processors import TakeFirst, MapCompose, Join from scrapy.loader import ItemLoader - from scrapy.loader.processors import TakeFirst, MapCompose, Join class ProductLoader(ItemLoader): @@ -214,7 +218,7 @@ output processors to use: in the :ref:`Item Field <topics-items-fields>` metadata. Here is an example:: import scrapy - from scrapy.loader.processors import Join, MapCompose, TakeFirst + from itemloaders.processors import Join, MapCompose, TakeFirst from w3lib.html import remove_tags def filter_price(value): @@ -295,250 +299,9 @@ There are several ways to modify Item Loader context values: ItemLoader objects ================== -.. class:: ItemLoader([item, selector, response], **kwargs) - - Return a new Item Loader for populating the given :ref:`item object - <topics-items>`. If no item object is given, one is instantiated - automatically using the class in :attr:`default_item_class`. - - When instantiated with a ``selector`` or a ``response`` parameters - the :class:`ItemLoader` class provides convenient mechanisms for extracting - data from web pages using :ref:`selectors <topics-selectors>`. - - :param item: The item instance to populate using subsequent calls to - :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`, - or :meth:`~ItemLoader.add_value`. - :type item: :ref:`item object <topics-items>` - - :param selector: The selector to extract data from, when using the - :meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath` - (resp. :meth:`replace_css`) method. - :type selector: :class:`~scrapy.selector.Selector` object - - :param response: The response used to construct the selector using the - :attr:`default_selector_class`, unless the selector argument is given, - in which case this argument is ignored. - :type response: :class:`~scrapy.http.Response` object - - The item, selector, response and the remaining keyword arguments are - assigned to the Loader context (accessible through the :attr:`context` attribute). - - :class:`ItemLoader` instances have the following methods: - - .. method:: get_value(value, *processors, **kwargs) - - Process the given ``value`` by the given ``processors`` and keyword - arguments. - - Available keyword arguments: - - :param re: a regular expression to use for extracting data from the - given value using :meth:`~scrapy.utils.misc.extract_regex` method, - applied before processors - :type re: str or compiled regex - - Examples: - - >>> from scrapy.loader.processors import TakeFirst - >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)') - 'FOO` - - .. method:: add_value(field_name, value, *processors, **kwargs) - - Process and then add the given ``value`` for the given field. - - The value is first passed through :meth:`get_value` by giving the - ``processors`` and ``kwargs``, and then passed through the - :ref:`field input processor <topics-loaders-processors>` and its result - appended to the data collected for that field. If the field already - contains collected data, the new data is added. - - The given ``field_name`` can be ``None``, in which case values for - multiple fields may be added. And the processed value should be a dict - with field_name mapped to values. - - Examples:: - - loader.add_value('name', u'Color TV') - loader.add_value('colours', [u'white', u'blue']) - loader.add_value('length', u'100') - loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)') - loader.add_value(None, {'name': u'foo', 'sex': u'male'}) - - .. method:: replace_value(field_name, value, *processors, **kwargs) - - Similar to :meth:`add_value` but replaces the collected data with the - new value instead of adding it. - .. method:: get_xpath(xpath, *processors, **kwargs) - - Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a - value, which is used to extract a list of unicode strings from the - selector associated with this :class:`ItemLoader`. - - :param xpath: the XPath to extract data from - :type xpath: str - - :param re: a regular expression to use for extracting data from the - selected XPath region - :type re: str or compiled regex - - Examples:: - - # HTML snippet: <p class="product-name">Color TV</p> - loader.get_xpath('//p[@class="product-name"]') - # HTML snippet: <p id="price">the price is $1200</p> - loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)') - - .. method:: add_xpath(field_name, xpath, *processors, **kwargs) - - Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a - value, which is used to extract a list of unicode strings from the - selector associated with this :class:`ItemLoader`. - - See :meth:`get_xpath` for ``kwargs``. - - :param xpath: the XPath to extract data from - :type xpath: str - - Examples:: - - # HTML snippet: <p class="product-name">Color TV</p> - loader.add_xpath('name', '//p[@class="product-name"]') - # HTML snippet: <p id="price">the price is $1200</p> - loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)') - - .. method:: replace_xpath(field_name, xpath, *processors, **kwargs) - - Similar to :meth:`add_xpath` but replaces collected data instead of - adding it. - - .. method:: get_css(css, *processors, **kwargs) - - Similar to :meth:`ItemLoader.get_value` but receives a CSS selector - instead of a value, which is used to extract a list of unicode strings - from the selector associated with this :class:`ItemLoader`. - - :param css: the CSS selector to extract data from - :type css: str - - :param re: a regular expression to use for extracting data from the - selected CSS region - :type re: str or compiled regex - - Examples:: - - # HTML snippet: <p class="product-name">Color TV</p> - loader.get_css('p.product-name') - # HTML snippet: <p id="price">the price is $1200</p> - loader.get_css('p#price', TakeFirst(), re='the price is (.*)') - - .. method:: add_css(field_name, css, *processors, **kwargs) - - Similar to :meth:`ItemLoader.add_value` but receives a CSS selector - instead of a value, which is used to extract a list of unicode strings - from the selector associated with this :class:`ItemLoader`. - - See :meth:`get_css` for ``kwargs``. - - :param css: the CSS selector to extract data from - :type css: str - - Examples:: - - # HTML snippet: <p class="product-name">Color TV</p> - loader.add_css('name', 'p.product-name') - # HTML snippet: <p id="price">the price is $1200</p> - loader.add_css('price', 'p#price', re='the price is (.*)') - - .. method:: replace_css(field_name, css, *processors, **kwargs) - - Similar to :meth:`add_css` but replaces collected data instead of - adding it. - - .. method:: load_item() - - Populate the item with the data collected so far, and return it. The - data collected is first passed through the :ref:`output processors - <topics-loaders-processors>` to get the final value to assign to each - item field. - - .. method:: nested_xpath(xpath) - - Create a nested loader with an xpath selector. - The supplied selector is applied relative to selector associated - with this :class:`ItemLoader`. The nested loader shares the :ref:`item - object <topics-items>` with the parent :class:`ItemLoader` so calls to - :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will - behave as expected. - - .. method:: nested_css(css) - - Create a nested loader with a css selector. - The supplied selector is applied relative to selector associated - with this :class:`ItemLoader`. The nested loader shares the :ref:`item - object <topics-items>` with the parent :class:`ItemLoader` so calls to - :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will - behave as expected. - - .. method:: get_collected_values(field_name) - - Return the collected values for the given field. - - .. method:: get_output_value(field_name) - - Return the collected values parsed using the output processor, for the - given field. This method doesn't populate or modify the item at all. - - .. method:: get_input_processor(field_name) - - Return the input processor for the given field. - - .. method:: get_output_processor(field_name) - - Return the output processor for the given field. - - :class:`ItemLoader` instances have the following attributes: - - .. attribute:: item - - The :ref:`item object <topics-items>` being parsed by this Item Loader. - This is mostly used as a property so when attempting to override this - value, you may want to check out :attr:`default_item_class` first. - - .. attribute:: context - - The currently active :ref:`Context <topics-loaders-context>` of this - Item Loader. - - .. attribute:: default_item_class - - An :ref:`item object <topics-items>` class or factory, used to - instantiate items when not given in the ``__init__`` method. - - .. attribute:: default_input_processor - - The default input processor to use for those fields which don't specify - one. - - .. attribute:: default_output_processor - - The default output processor to use for those fields which don't specify - one. - - .. attribute:: default_selector_class - - The class used to construct the :attr:`selector` of this - :class:`ItemLoader`, if only a response is given in the ``__init__`` method. - If a selector is given in the ``__init__`` method this attribute is ignored. - This attribute is sometimes overridden in subclasses. - - .. attribute:: selector - - The :class:`~scrapy.selector.Selector` object to extract data from. - It's either the selector given in the ``__init__`` method or one created from - the response given in the ``__init__`` method using the - :attr:`default_selector_class`. This attribute is meant to be - read-only. +.. autoclass:: scrapy.loader.ItemLoader + :members: + :inherited-members: .. _topics-loaders-nested: @@ -609,7 +372,7 @@ those dashes in the final product names. Here's how you can remove those dashes by reusing and extending the default Product Item Loader (``ProductLoader``):: - from scrapy.loader.processors import MapCompose + from itemloaders.processors import MapCompose from myproject.ItemLoaders import ProductLoader def strip_dashes(x): @@ -622,7 +385,7 @@ Another case where extending Item Loaders can be very helpful is when you have multiple source formats, for example XML and HTML. In the XML version you may want to remove ``CDATA`` occurrences. Here's an example of how to do it:: - from scrapy.loader.processors import MapCompose + from itemloaders.processors import MapCompose from myproject.ItemLoaders import ProductLoader from myproject.utils.xml import remove_cdata @@ -642,156 +405,5 @@ projects. Scrapy only provides the mechanism; it doesn't impose any specific organization of your Loaders collection - that's up to you and your project's needs. -.. _topics-loaders-available-processors: - -Available built-in processors -============================= - -.. module:: scrapy.loader.processors - :synopsis: A collection of processors to use with Item Loaders - -Even though you can use any callable function as input and output processors, -Scrapy provides some commonly used processors, which are described below. Some -of them, like the :class:`MapCompose` (which is typically used as input -processor) compose the output of several functions executed in order, to -produce the final parsed value. - -Here is a list of all built-in processors: - -.. class:: Identity - - The simplest processor, which doesn't do anything. It returns the original - values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it - accept Loader contexts. - - Example: - - >>> from scrapy.loader.processors import Identity - >>> proc = Identity() - >>> proc(['one', 'two', 'three']) - ['one', 'two', 'three'] - -.. class:: TakeFirst - - Returns the first non-null/non-empty value from the values received, - so it's typically used as an output processor to single-valued fields. - It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts. - - Example: - - >>> from scrapy.loader.processors import TakeFirst - >>> proc = TakeFirst() - >>> proc(['', 'one', 'two', 'three']) - 'one' - -.. class:: Join(separator=u' ') - - Returns the values joined with the separator given in the ``__init__`` method, which - defaults to ``u' '``. It doesn't accept Loader contexts. - - When using the default separator, this processor is equivalent to the - function: ``u' '.join`` - - Examples: - - >>> from scrapy.loader.processors import Join - >>> proc = Join() - >>> proc(['one', 'two', 'three']) - 'one two three' - >>> proc = Join('<br>') - >>> proc(['one', 'two', 'three']) - 'one<br>two<br>three' - -.. class:: Compose(*functions, **default_loader_context) - - A processor which is constructed from the composition of the given - functions. This means that each input value of this processor is passed to - the first function, and the result of that function is passed to the second - function, and so on, until the last function returns the output value of - this processor. - - By default, stop process on ``None`` value. This behaviour can be changed by - passing keyword argument ``stop_on_none=False``. - - Example: - - >>> from scrapy.loader.processors import Compose - >>> proc = Compose(lambda v: v[0], str.upper) - >>> proc(['hello', 'world']) - 'HELLO' - - Each function can optionally receive a ``loader_context`` parameter. For - those which do, this processor will pass the currently active :ref:`Loader - context <topics-loaders-context>` through that parameter. - - The keyword arguments passed in the ``__init__`` method are used as the default - Loader context values passed to each function call. However, the final - Loader context values passed to functions are overridden with the currently - active Loader context accessible through the :meth:`ItemLoader.context` - attribute. - -.. class:: MapCompose(*functions, **default_loader_context) - - A processor which is constructed from the composition of the given - functions, similar to the :class:`Compose` processor. The difference with - this processor is the way internal results are passed among functions, - which is as follows: - - The input value of this processor is *iterated* and the first function is - applied to each element. The results of these function calls (one for each element) - are concatenated to construct a new iterable, which is then used to apply the - second function, and so on, until the last function is applied to each - value of the list of values collected so far. The output values of the last - function are concatenated together to produce the output of this processor. - - Each particular function can return a value or a list of values, which is - flattened with the list of values returned by the same function applied to - the other input values. The functions can also return ``None`` in which - case the output of that function is ignored for further processing over the - chain. - - This processor provides a convenient way to compose functions that only - work with single values (instead of iterables). For this reason the - :class:`MapCompose` processor is typically used as input processor, since - data is often extracted using the - :meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors - <topics-selectors>`, which returns a list of unicode strings. - - The example below should clarify how it works: - - >>> def filter_world(x): - ... return None if x == 'world' else x - ... - >>> from scrapy.loader.processors import MapCompose - >>> proc = MapCompose(filter_world, str.upper) - >>> proc(['hello', 'world', 'this', 'is', 'scrapy']) - ['HELLO, 'THIS', 'IS', 'SCRAPY'] - - As with the Compose processor, functions can receive Loader contexts, and - ``__init__`` method keyword arguments are used as default context values. See - :class:`Compose` processor for more info. - -.. class:: SelectJmes(json_path) - - Queries the value using the json path provided to the ``__init__`` method and returns the output. - Requires jmespath (https://github.com/jmespath/jmespath.py) to run. - This processor takes only one input at a time. - - Example: - - >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose - >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries - >>> proc({'foo': 'bar'}) - 'bar' - >>> proc({'foo': {'bar': 'baz'}}) - {'bar': 'baz'} - - Working with Json: - - >>> import json - >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo")) - >>> proc_single_json_str('{"foo": "bar"}') - 'bar' - >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo'))) - >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]') - ['bar'] +.. _itemloaders: https://itemloaders.readthedocs.io/en/latest/ +.. _processors: https://itemloaders.readthedocs.io/en/latest/built-in-processors.html diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 18f57945f..014951a8e 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -3,217 +3,86 @@ Item Loader See documentation in docs/topics/loaders.rst """ -from collections import defaultdict -from contextlib import suppress - -from itemadapter import ItemAdapter +import itemloaders from scrapy.item import Item -from scrapy.loader.common import wrap_loader_context -from scrapy.loader.processors import Identity from scrapy.selector import Selector -from scrapy.utils.misc import arg_to_iter, extract_regex -from scrapy.utils.python import flatten -def unbound_method(method): +class ItemLoader(itemloaders.ItemLoader): """ - Allow to use single-argument functions as input or output processors - (no need to define an unused first 'self' argument) + A user-friendly abstraction to populate an :ref:`item <topics-items>` with data + by applying :ref:`field processors <topics-loaders-processors>` to scraped data. + When instantiated with a ``selector`` or a ``response`` it supports + data extraction from web pages using :ref:`selectors <topics-selectors>`. + + :param item: The item instance to populate using subsequent calls to + :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`, + or :meth:`~ItemLoader.add_value`. + :type item: scrapy.item.Item + + :param selector: The selector to extract data from, when using the + :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or + :meth:`replace_css` method. + :type selector: :class:`~scrapy.selector.Selector` object + + :param response: The response used to construct the selector using the + :attr:`default_selector_class`, unless the selector argument is given, + in which case this argument is ignored. + :type response: :class:`~scrapy.http.Response` object + + If no item is given, one is instantiated automatically using the class in + :attr:`default_item_class`. + + The item, selector, response and remaining keyword arguments are + assigned to the Loader context (accessible through the :attr:`context` attribute). + + .. attribute:: item + + The item object being parsed by this Item Loader. + This is mostly used as a property so, when attempting to override this + value, you may want to check out :attr:`default_item_class` first. + + .. attribute:: context + + The currently active :ref:`Context <loaders-context>` of this Item Loader. + + .. attribute:: default_item_class + + An :ref:`item <topics-items>` class (or factory), used to instantiate + items when not given in the ``__init__`` method. + + .. attribute:: default_input_processor + + The default input processor to use for those fields which don't specify + one. + + .. attribute:: default_output_processor + + The default output processor to use for those fields which don't specify + one. + + .. attribute:: default_selector_class + + The class used to construct the :attr:`selector` of this + :class:`ItemLoader`, if only a response is given in the ``__init__`` method. + If a selector is given in the ``__init__`` method this attribute is ignored. + This attribute is sometimes overridden in subclasses. + + .. attribute:: selector + + The :class:`~scrapy.selector.Selector` object to extract data from. + It's either the selector given in the ``__init__`` method or one created from + the response given in the ``__init__`` method using the + :attr:`default_selector_class`. This attribute is meant to be + read-only. """ - with suppress(AttributeError): - if '.' not in method.__qualname__: - return method.__func__ - return method - - -class ItemLoader: default_item_class = Item - default_input_processor = Identity() - default_output_processor = Identity() default_selector_class = Selector def __init__(self, item=None, selector=None, response=None, parent=None, **context): if selector is None and response is not None: selector = self.default_selector_class(response) - self.selector = selector - context.update(selector=selector, response=response) - if item is None: - item = self.default_item_class() - self.context = context - self.parent = parent - self._local_item = context['item'] = item - self._local_values = defaultdict(list) - # values from initial item - for field_name, value in ItemAdapter(item).items(): - self._values[field_name] += arg_to_iter(value) - - @property - def _values(self): - if self.parent is not None: - return self.parent._values - else: - return self._local_values - - @property - def item(self): - if self.parent is not None: - return self.parent.item - else: - return self._local_item - - def nested_xpath(self, xpath, **context): - selector = self.selector.xpath(xpath) - context.update(selector=selector) - subloader = self.__class__( - item=self.item, parent=self, **context - ) - return subloader - - def nested_css(self, css, **context): - selector = self.selector.css(css) - context.update(selector=selector) - subloader = self.__class__( - item=self.item, parent=self, **context - ) - return subloader - - def add_value(self, field_name, value, *processors, **kw): - value = self.get_value(value, *processors, **kw) - if value is None: - return - if not field_name: - for k, v in value.items(): - self._add_value(k, v) - else: - self._add_value(field_name, value) - - def replace_value(self, field_name, value, *processors, **kw): - value = self.get_value(value, *processors, **kw) - if value is None: - return - if not field_name: - for k, v in value.items(): - self._replace_value(k, v) - else: - self._replace_value(field_name, value) - - def _add_value(self, field_name, value): - value = arg_to_iter(value) - processed_value = self._process_input_value(field_name, value) - if processed_value: - self._values[field_name] += arg_to_iter(processed_value) - - def _replace_value(self, field_name, value): - self._values.pop(field_name, None) - self._add_value(field_name, value) - - def get_value(self, value, *processors, **kw): - regex = kw.get('re', None) - if regex: - value = arg_to_iter(value) - value = flatten(extract_regex(regex, x) for x in value) - - for proc in processors: - if value is None: - break - _proc = proc - proc = wrap_loader_context(proc, self.context) - try: - value = proc(value) - except Exception as e: - raise ValueError("Error with processor %s value=%r error='%s: %s'" % - (_proc.__class__.__name__, value, - type(e).__name__, str(e))) - return value - - def load_item(self): - adapter = ItemAdapter(self.item) - for field_name in tuple(self._values): - value = self.get_output_value(field_name) - if value is not None: - adapter[field_name] = value - return adapter.item - - def get_output_value(self, field_name): - proc = self.get_output_processor(field_name) - proc = wrap_loader_context(proc, self.context) - try: - return proc(self._values[field_name]) - except Exception as e: - raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % - (field_name, self._values[field_name], type(e).__name__, str(e))) - - def get_collected_values(self, field_name): - return self._values[field_name] - - def get_input_processor(self, field_name): - proc = getattr(self, '%s_in' % field_name, None) - if not proc: - proc = self._get_item_field_attr(field_name, 'input_processor', - self.default_input_processor) - return unbound_method(proc) - - def get_output_processor(self, field_name): - proc = getattr(self, '%s_out' % field_name, None) - if not proc: - proc = self._get_item_field_attr(field_name, 'output_processor', - self.default_output_processor) - return unbound_method(proc) - - def _process_input_value(self, field_name, value): - proc = self.get_input_processor(field_name) - _proc = proc - proc = wrap_loader_context(proc, self.context) - try: - return proc(value) - except Exception as e: - raise ValueError( - "Error with input processor %s: field=%r value=%r " - "error='%s: %s'" % (_proc.__class__.__name__, field_name, - value, type(e).__name__, str(e))) - - def _get_item_field_attr(self, field_name, key, default=None): - field_meta = ItemAdapter(self.item).get_field_meta(field_name) - return field_meta.get(key, default) - - def _check_selector_method(self): - if self.selector is None: - raise RuntimeError("To use XPath or CSS selectors, " - "%s must be instantiated with a selector " - "or a response" % self.__class__.__name__) - - def add_xpath(self, field_name, xpath, *processors, **kw): - values = self._get_xpathvalues(xpath, **kw) - self.add_value(field_name, values, *processors, **kw) - - def replace_xpath(self, field_name, xpath, *processors, **kw): - values = self._get_xpathvalues(xpath, **kw) - self.replace_value(field_name, values, *processors, **kw) - - def get_xpath(self, xpath, *processors, **kw): - values = self._get_xpathvalues(xpath, **kw) - return self.get_value(values, *processors, **kw) - - def _get_xpathvalues(self, xpaths, **kw): - self._check_selector_method() - xpaths = arg_to_iter(xpaths) - return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths) - - def add_css(self, field_name, css, *processors, **kw): - values = self._get_cssvalues(css, **kw) - self.add_value(field_name, values, *processors, **kw) - - def replace_css(self, field_name, css, *processors, **kw): - values = self._get_cssvalues(css, **kw) - self.replace_value(field_name, values, *processors, **kw) - - def get_css(self, css, *processors, **kw): - values = self._get_cssvalues(css, **kw) - return self.get_value(values, *processors, **kw) - - def _get_cssvalues(self, csss, **kw): - self._check_selector_method() - csss = arg_to_iter(csss) - return flatten(self.selector.css(css).getall() for css in csss) + context.update(response=response) + super().__init__(item=item, selector=selector, parent=parent, **context) diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py index 42f8de636..3b8a6ee94 100644 --- a/scrapy/loader/common.py +++ b/scrapy/loader/common.py @@ -1,14 +1,21 @@ """Common functions used in Item Loaders code""" -from functools import partial -from scrapy.utils.python import get_func_args +import warnings + +from itemloaders import common + +from scrapy.utils.deprecate import ScrapyDeprecationWarning def wrap_loader_context(function, context): """Wrap functions that receive loader_context to contain the context "pre-loaded" and expose a interface that receives only one argument """ - if 'loader_context' in get_func_args(function): - return partial(function, loader_context=context) - else: - return function + warnings.warn( + "scrapy.loader.common.wrap_loader_context has moved to a new library." + "Please update your reference to itemloaders.common.wrap_loader_context", + ScrapyDeprecationWarning, + stacklevel=2 + ) + + return common.wrap_loader_context(function, context) diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py index a7be65609..51fbd19eb 100644 --- a/scrapy/loader/processors.py +++ b/scrapy/loader/processors.py @@ -3,102 +3,19 @@ This module provides some commonly used processors for Item Loaders. See documentation in docs/topics/loaders.rst """ -from collections import ChainMap +from itemloaders import processors -from scrapy.utils.misc import arg_to_iter -from scrapy.loader.common import wrap_loader_context +from scrapy.utils.deprecate import create_deprecated_class -class MapCompose: +MapCompose = create_deprecated_class('MapCompose', processors.MapCompose) - def __init__(self, *functions, **default_loader_context): - self.functions = functions - self.default_loader_context = default_loader_context +Compose = create_deprecated_class('Compose', processors.Compose) - def __call__(self, value, loader_context=None): - values = arg_to_iter(value) - if loader_context: - context = ChainMap(loader_context, self.default_loader_context) - else: - context = self.default_loader_context - wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] - for func in wrapped_funcs: - next_values = [] - for v in values: - try: - next_values += arg_to_iter(func(v)) - except Exception as e: - raise ValueError("Error in MapCompose with " - "%s value=%r error='%s: %s'" % - (str(func), value, type(e).__name__, - str(e))) - values = next_values - return values +TakeFirst = create_deprecated_class('TakeFirst', processors.TakeFirst) +Identity = create_deprecated_class('Identity', processors.Identity) -class Compose: +SelectJmes = create_deprecated_class('SelectJmes', processors.SelectJmes) - def __init__(self, *functions, **default_loader_context): - self.functions = functions - self.stop_on_none = default_loader_context.get('stop_on_none', True) - self.default_loader_context = default_loader_context - - def __call__(self, value, loader_context=None): - if loader_context: - context = ChainMap(loader_context, self.default_loader_context) - else: - context = self.default_loader_context - wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] - for func in wrapped_funcs: - if value is None and self.stop_on_none: - break - try: - value = func(value) - except Exception as e: - raise ValueError("Error in Compose with " - "%s value=%r error='%s: %s'" % - (str(func), value, type(e).__name__, str(e))) - return value - - -class TakeFirst: - - def __call__(self, values): - for value in values: - if value is not None and value != '': - return value - - -class Identity: - - def __call__(self, values): - return values - - -class SelectJmes: - """ - Query the input string for the jmespath (given at instantiation), - and return the answer - Requires : jmespath(https://github.com/jmespath/jmespath) - Note: SelectJmes accepts only one input element at a time. - """ - def __init__(self, json_path): - self.json_path = json_path - import jmespath - self.compiled_path = jmespath.compile(self.json_path) - - def __call__(self, value): - """Query value for the jmespath query and return answer - :param value: a data structure (dict, list) to extract from - :return: Element extracted according to jmespath query - """ - return self.compiled_path.search(value) - - -class Join: - - def __init__(self, separator=u' '): - self.separator = separator - - def __call__(self, values): - return self.separator.join(values) +Join = create_deprecated_class('Join', processors.Join) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 8e5fde246..d6966be8e 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -15,6 +15,7 @@ from w3lib.html import replace_entities from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.python import flatten, to_unicode from scrapy.item import _BaseItem +from scrapy.utils.deprecate import ScrapyDeprecationWarning _ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes @@ -86,6 +87,11 @@ def extract_regex(regex, text, encoding='utf-8'): * if the regex contains multiple numbered groups, all those will be returned (flattened) * if the regex doesn't contain any group the entire regex matching is returned """ + warnings.warn( + "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.", + ScrapyDeprecationWarning, + stacklevel=2 + ) if isinstance(regex, str): regex = re.compile(regex, re.UNICODE) diff --git a/setup.cfg b/setup.cfg index a9138c1c0..46a3d13fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -118,6 +118,9 @@ ignore_errors = True [mypy-tests.test_loader] ignore_errors = True +[mypy-tests.test_loader_deprecated] +ignore_errors = True + [mypy-tests.test_pipeline_crawl] ignore_errors = True diff --git a/setup.py b/setup.py index 5a99fd1bf..f8d9b491b 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,7 @@ setup( 'Twisted>=17.9.0', 'cryptography>=2.0', 'cssselect>=0.9.1', + 'itemloaders>=1.0.1', 'lxml>=3.5.0', 'parsel>=1.5.0', 'PyDispatcher>=2.0.5', diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index dacb86e56..0551b1e95 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,7 +1,6 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -jmespath mitmproxy; python_version >= '3.6' mitmproxy<4.0.0; python_version < '3.6' # https://github.com/pytest-dev/pytest-twisted/issues/93 diff --git a/tests/test_loader.py b/tests/test_loader.py index 8a9c6fca9..581183625 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,14 +1,12 @@ -from functools import partial import unittest import attr from itemadapter import ItemAdapter +from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst from scrapy.http import HtmlResponse from scrapy.item import Item, Field from scrapy.loader import ItemLoader -from scrapy.loader.processors import (Compose, Identity, Join, - MapCompose, SelectJmes, TakeFirst) from scrapy.selector import Selector @@ -69,6 +67,10 @@ def processor_with_args(value, other=None, loader_context=None): class BasicItemLoaderTest(unittest.TestCase): + def test_add_value_on_unknown_field(self): + il = TestItemLoader() + self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo']) + def test_load_item_using_default_loader(self): i = TestItem() i['summary'] = u'lala' @@ -85,391 +87,6 @@ class BasicItemLoaderTest(unittest.TestCase): item = il.load_item() self.assertEqual(item['name'], [u'Marta']) - def test_load_item_ignore_none_field_values(self): - def validate_sku(value): - # Let's assume a SKU is only digits. - if value.isdigit(): - return value - - class MyLoader(ItemLoader): - name_out = Compose(lambda vs: vs[0]) # take first which allows empty values - price_out = Compose(TakeFirst(), float) - sku_out = Compose(TakeFirst(), validate_sku) - - valid_fragment = u'SKU: 1234' - invalid_fragment = u'SKU: not available' - sku_re = 'SKU: (.+)' - - il = MyLoader(item={}) - # Should not return "sku: None". - il.add_value('sku', [invalid_fragment], re=sku_re) - # Should not ignore empty values. - il.add_value('name', u'') - il.add_value('price', [u'0']) - self.assertEqual(il.load_item(), { - 'name': u'', - 'price': 0.0, - }) - - il.replace_value('sku', [valid_fragment], re=sku_re) - self.assertEqual(il.load_item()['sku'], u'1234') - - def test_self_referencing_loader(self): - class MyLoader(ItemLoader): - url_out = TakeFirst() - - def img_url_out(self, values): - return (self.get_output_value('url') or '') + values[0] - - il = MyLoader(item={}) - il.add_value('url', 'http://example.com/') - il.add_value('img_url', '1234.png') - self.assertEqual(il.load_item(), { - 'url': 'http://example.com/', - 'img_url': 'http://example.com/1234.png', - }) - - il = MyLoader(item={}) - il.add_value('img_url', '1234.png') - self.assertEqual(il.load_item(), { - 'img_url': '1234.png', - }) - - def test_add_value(self): - il = TestItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'Marta']) - self.assertEqual(il.get_output_value('name'), [u'Marta']) - il.add_value('name', u'pepe') - self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe']) - self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe']) - - # test add object value - il.add_value('summary', {'key': 1}) - self.assertEqual(il.get_collected_values('summary'), [{'key': 1}]) - - il.add_value(None, u'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim']) - - def test_add_zero(self): - il = NameItemLoader() - il.add_value('name', 0) - self.assertEqual(il.get_collected_values('name'), [0]) - - def test_replace_value(self): - il = TestItemLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'Marta']) - self.assertEqual(il.get_output_value('name'), [u'Marta']) - il.replace_value('name', u'pepe') - self.assertEqual(il.get_collected_values('name'), [u'Pepe']) - self.assertEqual(il.get_output_value('name'), [u'Pepe']) - - il.replace_value(None, u'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), [u'Jim']) - - def test_get_value(self): - il = NameItemLoader() - self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper)) - self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$')) - self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')) - - il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$') - self.assertEqual([u'foo'], il.get_collected_values('name')) - il.replace_value('name', u'name:bar', re=u'name:(.*)$') - self.assertEqual([u'bar'], il.get_collected_values('name')) - - def test_iter_on_input_processor_input(self): - class NameFirstItemLoader(NameItemLoader): - name_in = TakeFirst() - - il = NameFirstItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'marta']) - il = NameFirstItemLoader() - il.add_value('name', [u'marta', u'jose']) - self.assertEqual(il.get_collected_values('name'), [u'marta']) - - il = NameFirstItemLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'marta']) - il = NameFirstItemLoader() - il.replace_value('name', [u'marta', u'jose']) - self.assertEqual(il.get_collected_values('name'), [u'marta']) - - il = NameFirstItemLoader() - il.add_value('name', u'marta') - il.add_value('name', [u'jose', u'pedro']) - self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose']) - - def test_map_compose_filter(self): - def filter_world(x): - return None if x == 'world' else x - - proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), - ['HELLO', 'THIS', 'IS', 'SCRAPY']) - - def test_map_compose_filter_multil(self): - class TestItemLoader(NameItemLoader): - name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) - - il = TestItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Mart']) - item = il.load_item() - self.assertEqual(item['name'], [u'Mart']) - - def test_default_input_processor(self): - il = DefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mart']) - - def test_inherited_default_input_processor(self): - class InheritDefaultedItemLoader(DefaultedItemLoader): - pass - - il = InheritDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mart']) - - def test_input_processor_inheritance(self): - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(lambda v: v.lower()) - - il = ChildItemLoader() - il.add_value('url', u'HTTP://scrapy.ORG') - self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org']) - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Marta']) - - class ChildChildItemLoader(ChildItemLoader): - url_in = MapCompose(lambda v: v.upper()) - summary_in = MapCompose(lambda v: v) - - il = ChildChildItemLoader() - il.add_value('url', u'http://scrapy.org') - self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG']) - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Marta']) - - def test_empty_map_compose(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose() - - il = IdentityDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'marta']) - - def test_identity_input_processor(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = Identity() - - il = IdentityDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'marta']) - - def test_extend_custom_input_processors(self): - class ChildItemLoader(TestItemLoader): - name_in = MapCompose(TestItemLoader.name_in, str.swapcase) - - il = ChildItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mARTA']) - - def test_extend_default_input_processors(self): - class ChildDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase) - - il = ChildDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'MART']) - - def test_output_processor_using_function(self): - il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) - - class TakeFirstItemLoader(TestItemLoader): - name_out = u" ".join - - il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar Ta') - - def test_output_processor_error(self): - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_out = MapCompose(float) - - il = TestItemLoader() - il.add_value('name', [u'$10']) - try: - float(u'$10') - except Exception as e: - expected_exc_str = str(e) - - exc = None - try: - il.load_item() - except Exception as e: - exc = e - assert isinstance(exc, ValueError) - s = str(exc) - assert 'name' in s, s - assert '$10' in s, s - assert 'ValueError' in s, s - assert expected_exc_str in s, s - - def test_output_processor_using_classes(self): - il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) - - class TakeFirstItemLoader(TestItemLoader): - name_out = Join() - - il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar Ta') - - class TakeFirstItemLoader(TestItemLoader): - name_out = Join("<br>") - - il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta') - - def test_default_output_processor(self): - il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) - - class LalaItemLoader(TestItemLoader): - default_output_processor = Identity() - - il = LalaItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) - - def test_loader_context_on_declaration(self): - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args, key=u'val') - - il = ChildItemLoader() - il.add_value('url', u'text') - self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') - self.assertEqual(il.get_output_value('url'), ['val']) - - def test_loader_context_on_instantiation(self): - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args) - - il = ChildItemLoader(key=u'val') - il.add_value('url', u'text') - self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') - self.assertEqual(il.get_output_value('url'), ['val']) - - def test_loader_context_on_assign(self): - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args) - - il = ChildItemLoader() - il.context['key'] = u'val' - il.add_value('url', u'text') - self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') - self.assertEqual(il.get_output_value('url'), ['val']) - - def test_item_passed_to_input_processor_functions(self): - def processor(value, loader_context): - return loader_context['item']['name'] - - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor) - - it = TestItem(name='marta') - il = ChildItemLoader(item=it) - il.add_value('url', u'text') - self.assertEqual(il.get_output_value('url'), ['marta']) - il.replace_value('url', u'text2') - self.assertEqual(il.get_output_value('url'), ['marta']) - - def test_add_value_on_unknown_field(self): - il = TestItemLoader() - self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo']) - - def test_compose_processor(self): - class TestItemLoader(NameItemLoader): - name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) - - il = TestItemLoader() - il.add_value('name', [u'marta', u'other']) - self.assertEqual(il.get_output_value('name'), u'Mart') - item = il.load_item() - self.assertEqual(item['name'], u'Mart') - - def test_partial_processor(self): - def join(values, sep=None, loader_context=None, ignored=None): - if sep is not None: - return sep.join(values) - elif loader_context and 'sep' in loader_context: - return loader_context['sep'].join(values) - else: - return ''.join(values) - - class TestItemLoader(NameItemLoader): - name_out = Compose(partial(join, sep='+')) - url_out = Compose(partial(join, loader_context={'sep': '.'})) - summary_out = Compose(partial(join, ignored='foo')) - - il = TestItemLoader() - il.add_value('name', [u'rabbit', u'hole']) - il.add_value('url', [u'rabbit', u'hole']) - il.add_value('summary', [u'rabbit', u'hole']) - item = il.load_item() - self.assertEqual(item['name'], u'rabbit+hole') - self.assertEqual(item['url'], u'rabbit.hole') - self.assertEqual(item['summary'], u'rabbithole') - - def test_error_input_processor(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_in = MapCompose(float) - - il = TestItemLoader() - self.assertRaises(ValueError, il.add_value, 'name', - [u'marta', u'other']) - - def test_error_output_processor(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_out = Compose(Join(), float) - - il = TestItemLoader() - il.add_value('name', u'marta') - with self.assertRaises(ValueError): - il.load_item() - - def test_error_processor_as_argument(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - - il = TestItemLoader() - self.assertRaises(ValueError, il.add_value, 'name', - [u'marta', u'other'], Compose(float)) - class InitializationTestMixin: @@ -587,41 +204,6 @@ class BaseNoInputReprocessingLoader(ItemLoader): title_out = TakeFirst() -class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader): - default_item_class = dict - - -class NoInputReprocessingFromDictTest(unittest.TestCase): - """ - Loaders initialized from loaded items must not reprocess fields (dict instances) - """ - def test_avoid_reprocessing_with_initial_values_single(self): - il = NoInputReprocessingDictLoader(item=dict(title='foo')) - il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title='foo')) - self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo')) - - def test_avoid_reprocessing_with_initial_values_list(self): - il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar'])) - il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title='foo')) - self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo')) - - def test_avoid_reprocessing_without_initial_values_single(self): - il = NoInputReprocessingDictLoader() - il.add_value('title', 'foo') - il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title='FOO')) - self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO')) - - def test_avoid_reprocessing_without_initial_values_list(self): - il = NoInputReprocessingDictLoader() - il.add_value('title', ['foo', 'bar']) - il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title='FOO')) - self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO')) - - class NoInputReprocessingItem(Item): title = Field() @@ -661,25 +243,6 @@ class NoInputReprocessingFromItemTest(unittest.TestCase): self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'}) -class TestOutputProcessorDict(unittest.TestCase): - def test_output_processor(self): - - class TempDict(dict): - def __init__(self, *args, **kwargs): - super(TempDict, self).__init__(self, *args, **kwargs) - self.setdefault('temp', 0.3) - - class TempLoader(ItemLoader): - default_item_class = TempDict - default_input_processor = Identity() - default_output_processor = Compose(TakeFirst()) - - loader = TempLoader() - item = loader.load_item() - self.assertIsInstance(item, TempDict) - self.assertEqual(dict(item), {'temp': 0.3}) - - class TestOutputProcessorItem(unittest.TestCase): def test_output_processor(self): @@ -701,49 +264,6 @@ class TestOutputProcessorItem(unittest.TestCase): self.assertEqual(dict(item), {'temp': 0.3}) -class ProcessorsTest(unittest.TestCase): - - def test_take_first(self): - proc = TakeFirst() - self.assertEqual(proc([None, '', 'hello', 'world']), 'hello') - self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0) - - def test_identity(self): - proc = Identity() - self.assertEqual(proc([None, '', 'hello', 'world']), - [None, '', 'hello', 'world']) - - def test_join(self): - proc = Join() - self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) - self.assertEqual(proc(['', 'hello', 'world']), u' hello world') - self.assertEqual(proc(['hello', 'world']), u'hello world') - self.assertIsInstance(proc(['hello', 'world']), str) - - def test_compose(self): - proc = Compose(lambda v: v[0], str.upper) - self.assertEqual(proc(['hello', 'world']), 'HELLO') - proc = Compose(str.upper) - self.assertEqual(proc(None), None) - proc = Compose(str.upper, stop_on_none=False) - self.assertRaises(ValueError, proc, None) - proc = Compose(str.upper, lambda x: x + 1) - self.assertRaises(ValueError, proc, 'hello') - - def test_mapcompose(self): - def filter_world(x): - return None if x == 'world' else x - proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), - [u'HELLO', u'THIS', u'IS', u'SCRAPY']) - proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc(None), []) - proc = MapCompose(filter_world, str.upper) - self.assertRaises(ValueError, proc, [1]) - proc = MapCompose(filter_world, lambda x: x + 1) - self.assertRaises(ValueError, proc, 'hello') - - class SelectortemLoaderTest(unittest.TestCase): response = HtmlResponse(url="", encoding='utf-8', body=b""" <html> @@ -921,6 +441,7 @@ class SubselectorLoaderTest(unittest.TestCase): def test_nested_xpath(self): l = NestedItemLoader(response=self.response) + nl = l.nested_xpath("//header") nl.add_xpath('name', 'div/text()') nl.add_css('name_div', '#id') @@ -998,31 +519,6 @@ class SubselectorLoaderTest(unittest.TestCase): self.assertEqual(item['image'], [u'/images/logo.png']) -class SelectJmesTestCase(unittest.TestCase): - test_list_equals = { - 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), - 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), - 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'dict': ( - 'foo.bar[*].name', - {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ['one', 'two'] - ), - 'list': ('[1]', [1, 2], 2) - } - - def test_output(self): - for l in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[l] - test = SelectJmes(expr)(test_list) - self.assertEqual( - test, - expected, - msg='test "{}" got {} expected {}'.format(l, test, expected) - ) - - # Functions as processors def function_processor_strip(iterable): @@ -1044,12 +540,6 @@ class FunctionProcessorItemLoader(ItemLoader): default_item_class = FunctionProcessorItem -class FunctionProcessorDictLoader(ItemLoader): - default_item_class = dict - foo_in = function_processor_strip - foo_out = function_processor_upper - - class FunctionProcessorTestCase(unittest.TestCase): def test_processor_defined_in_item(self): @@ -1061,15 +551,6 @@ class FunctionProcessorTestCase(unittest.TestCase): {'foo': ['BAR', 'ASDF', 'QWERTY']} ) - def test_processor_defined_in_item_loader(self): - lo = FunctionProcessorDictLoader() - lo.add_value('foo', ' bar ') - lo.add_value('foo', [' asdf ', ' qwerty ']) - self.assertEqual( - dict(lo.load_item()), - {'foo': ['BAR', 'ASDF', 'QWERTY']} - ) - if __name__ == "__main__": unittest.main() diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py new file mode 100644 index 000000000..d0a59e8cd --- /dev/null +++ b/tests/test_loader_deprecated.py @@ -0,0 +1,720 @@ +""" +These tests are kept as references from the ones that were ported to a itemloaders library. +Once we remove the references from scrapy, we can remove these tests. +""" + +import unittest +import warnings +from functools import partial + +from itemloaders.processors import (Compose, Identity, Join, + MapCompose, SelectJmes, TakeFirst) + +from scrapy.item import Item, Field +from scrapy.loader import ItemLoader +from scrapy.loader.common import wrap_loader_context +from scrapy.utils.deprecate import ScrapyDeprecationWarning +from scrapy.utils.misc import extract_regex + + +# test items +class NameItem(Item): + name = Field() + + +class TestItem(NameItem): + url = Field() + summary = Field() + + +# test item loaders +class NameItemLoader(ItemLoader): + default_item_class = TestItem + + +class TestItemLoader(NameItemLoader): + name_in = MapCompose(lambda v: v.title()) + + +class DefaultedItemLoader(NameItemLoader): + default_input_processor = MapCompose(lambda v: v[:-1]) + + +# test processors +def processor_with_args(value, other=None, loader_context=None): + if 'key' in loader_context: + return loader_context['key'] + return value + + +class BasicItemLoaderTest(unittest.TestCase): + + def test_load_item_using_default_loader(self): + i = TestItem() + i['summary'] = u'lala' + il = ItemLoader(item=i) + il.add_value('name', u'marta') + item = il.load_item() + assert item is i + self.assertEqual(item['summary'], [u'lala']) + self.assertEqual(item['name'], [u'marta']) + + def test_load_item_using_custom_loader(self): + il = TestItemLoader() + il.add_value('name', u'marta') + item = il.load_item() + self.assertEqual(item['name'], [u'Marta']) + + def test_load_item_ignore_none_field_values(self): + def validate_sku(value): + # Let's assume a SKU is only digits. + if value.isdigit(): + return value + + class MyLoader(ItemLoader): + name_out = Compose(lambda vs: vs[0]) # take first which allows empty values + price_out = Compose(TakeFirst(), float) + sku_out = Compose(TakeFirst(), validate_sku) + + valid_fragment = u'SKU: 1234' + invalid_fragment = u'SKU: not available' + sku_re = 'SKU: (.+)' + + il = MyLoader(item={}) + # Should not return "sku: None". + il.add_value('sku', [invalid_fragment], re=sku_re) + # Should not ignore empty values. + il.add_value('name', u'') + il.add_value('price', [u'0']) + self.assertEqual(il.load_item(), { + 'name': u'', + 'price': 0.0, + }) + + il.replace_value('sku', [valid_fragment], re=sku_re) + self.assertEqual(il.load_item()['sku'], u'1234') + + def test_self_referencing_loader(self): + class MyLoader(ItemLoader): + url_out = TakeFirst() + + def img_url_out(self, values): + return (self.get_output_value('url') or '') + values[0] + + il = MyLoader(item={}) + il.add_value('url', 'http://example.com/') + il.add_value('img_url', '1234.png') + self.assertEqual(il.load_item(), { + 'url': 'http://example.com/', + 'img_url': 'http://example.com/1234.png', + }) + + il = MyLoader(item={}) + il.add_value('img_url', '1234.png') + self.assertEqual(il.load_item(), { + 'img_url': '1234.png', + }) + + def test_add_value(self): + il = TestItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_collected_values('name'), [u'Marta']) + self.assertEqual(il.get_output_value('name'), [u'Marta']) + il.add_value('name', u'pepe') + self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe']) + self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe']) + + # test add object value + il.add_value('summary', {'key': 1}) + self.assertEqual(il.get_collected_values('summary'), [{'key': 1}]) + + il.add_value(None, u'Jim', lambda x: {'name': x}) + self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim']) + + def test_add_zero(self): + il = NameItemLoader() + il.add_value('name', 0) + self.assertEqual(il.get_collected_values('name'), [0]) + + def test_replace_value(self): + il = TestItemLoader() + il.replace_value('name', u'marta') + self.assertEqual(il.get_collected_values('name'), [u'Marta']) + self.assertEqual(il.get_output_value('name'), [u'Marta']) + il.replace_value('name', u'pepe') + self.assertEqual(il.get_collected_values('name'), [u'Pepe']) + self.assertEqual(il.get_output_value('name'), [u'Pepe']) + + il.replace_value(None, u'Jim', lambda x: {'name': x}) + self.assertEqual(il.get_collected_values('name'), [u'Jim']) + + def test_get_value(self): + il = NameItemLoader() + self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper)) + self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$')) + self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')) + + il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$') + self.assertEqual([u'foo'], il.get_collected_values('name')) + il.replace_value('name', u'name:bar', re=u'name:(.*)$') + self.assertEqual([u'bar'], il.get_collected_values('name')) + + def test_iter_on_input_processor_input(self): + class NameFirstItemLoader(NameItemLoader): + name_in = TakeFirst() + + il = NameFirstItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_collected_values('name'), [u'marta']) + il = NameFirstItemLoader() + il.add_value('name', [u'marta', u'jose']) + self.assertEqual(il.get_collected_values('name'), [u'marta']) + + il = NameFirstItemLoader() + il.replace_value('name', u'marta') + self.assertEqual(il.get_collected_values('name'), [u'marta']) + il = NameFirstItemLoader() + il.replace_value('name', [u'marta', u'jose']) + self.assertEqual(il.get_collected_values('name'), [u'marta']) + + il = NameFirstItemLoader() + il.add_value('name', u'marta') + il.add_value('name', [u'jose', u'pedro']) + self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose']) + + def test_map_compose_filter(self): + def filter_world(x): + return None if x == 'world' else x + + proc = MapCompose(filter_world, str.upper) + self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), + ['HELLO', 'THIS', 'IS', 'SCRAPY']) + + def test_map_compose_filter_multil(self): + class TestItemLoader(NameItemLoader): + name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) + + il = TestItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'Mart']) + item = il.load_item() + self.assertEqual(item['name'], [u'Mart']) + + def test_default_input_processor(self): + il = DefaultedItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'mart']) + + def test_inherited_default_input_processor(self): + class InheritDefaultedItemLoader(DefaultedItemLoader): + pass + + il = InheritDefaultedItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'mart']) + + def test_input_processor_inheritance(self): + class ChildItemLoader(TestItemLoader): + url_in = MapCompose(lambda v: v.lower()) + + il = ChildItemLoader() + il.add_value('url', u'HTTP://scrapy.ORG') + self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org']) + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'Marta']) + + class ChildChildItemLoader(ChildItemLoader): + url_in = MapCompose(lambda v: v.upper()) + summary_in = MapCompose(lambda v: v) + + il = ChildChildItemLoader() + il.add_value('url', u'http://scrapy.org') + self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG']) + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'Marta']) + + def test_empty_map_compose(self): + class IdentityDefaultedItemLoader(DefaultedItemLoader): + name_in = MapCompose() + + il = IdentityDefaultedItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'marta']) + + def test_identity_input_processor(self): + class IdentityDefaultedItemLoader(DefaultedItemLoader): + name_in = Identity() + + il = IdentityDefaultedItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'marta']) + + def test_extend_custom_input_processors(self): + class ChildItemLoader(TestItemLoader): + name_in = MapCompose(TestItemLoader.name_in, str.swapcase) + + il = ChildItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'mARTA']) + + def test_extend_default_input_processors(self): + class ChildDefaultedItemLoader(DefaultedItemLoader): + name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase) + + il = ChildDefaultedItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'MART']) + + def test_output_processor_using_function(self): + il = TestItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + + class TakeFirstItemLoader(TestItemLoader): + name_out = u" ".join + + il = TakeFirstItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), u'Mar Ta') + + def test_output_processor_error(self): + class TestItemLoader(ItemLoader): + default_item_class = TestItem + name_out = MapCompose(float) + + il = TestItemLoader() + il.add_value('name', [u'$10']) + try: + float(u'$10') + except Exception as e: + expected_exc_str = str(e) + + exc = None + try: + il.load_item() + except Exception as e: + exc = e + assert isinstance(exc, ValueError) + s = str(exc) + assert 'name' in s, s + assert '$10' in s, s + assert 'ValueError' in s, s + assert expected_exc_str in s, s + + def test_output_processor_using_classes(self): + il = TestItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + + class TakeFirstItemLoader(TestItemLoader): + name_out = Join() + + il = TakeFirstItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), u'Mar Ta') + + class TakeFirstItemLoader(TestItemLoader): + name_out = Join("<br>") + + il = TakeFirstItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta') + + def test_default_output_processor(self): + il = TestItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + + class LalaItemLoader(TestItemLoader): + default_output_processor = Identity() + + il = LalaItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + + def test_loader_context_on_declaration(self): + class ChildItemLoader(TestItemLoader): + url_in = MapCompose(processor_with_args, key=u'val') + + il = ChildItemLoader() + il.add_value('url', u'text') + self.assertEqual(il.get_output_value('url'), ['val']) + il.replace_value('url', u'text2') + self.assertEqual(il.get_output_value('url'), ['val']) + + def test_loader_context_on_instantiation(self): + class ChildItemLoader(TestItemLoader): + url_in = MapCompose(processor_with_args) + + il = ChildItemLoader(key=u'val') + il.add_value('url', u'text') + self.assertEqual(il.get_output_value('url'), ['val']) + il.replace_value('url', u'text2') + self.assertEqual(il.get_output_value('url'), ['val']) + + def test_loader_context_on_assign(self): + class ChildItemLoader(TestItemLoader): + url_in = MapCompose(processor_with_args) + + il = ChildItemLoader() + il.context['key'] = u'val' + il.add_value('url', u'text') + self.assertEqual(il.get_output_value('url'), ['val']) + il.replace_value('url', u'text2') + self.assertEqual(il.get_output_value('url'), ['val']) + + def test_item_passed_to_input_processor_functions(self): + def processor(value, loader_context): + return loader_context['item']['name'] + + class ChildItemLoader(TestItemLoader): + url_in = MapCompose(processor) + + it = TestItem(name='marta') + il = ChildItemLoader(item=it) + il.add_value('url', u'text') + self.assertEqual(il.get_output_value('url'), ['marta']) + il.replace_value('url', u'text2') + self.assertEqual(il.get_output_value('url'), ['marta']) + + def test_compose_processor(self): + class TestItemLoader(NameItemLoader): + name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) + + il = TestItemLoader() + il.add_value('name', [u'marta', u'other']) + self.assertEqual(il.get_output_value('name'), u'Mart') + item = il.load_item() + self.assertEqual(item['name'], u'Mart') + + def test_partial_processor(self): + def join(values, sep=None, loader_context=None, ignored=None): + if sep is not None: + return sep.join(values) + elif loader_context and 'sep' in loader_context: + return loader_context['sep'].join(values) + else: + return ''.join(values) + + class TestItemLoader(NameItemLoader): + name_out = Compose(partial(join, sep='+')) + url_out = Compose(partial(join, loader_context={'sep': '.'})) + summary_out = Compose(partial(join, ignored='foo')) + + il = TestItemLoader() + il.add_value('name', [u'rabbit', u'hole']) + il.add_value('url', [u'rabbit', u'hole']) + il.add_value('summary', [u'rabbit', u'hole']) + item = il.load_item() + self.assertEqual(item['name'], u'rabbit+hole') + self.assertEqual(item['url'], u'rabbit.hole') + self.assertEqual(item['summary'], u'rabbithole') + + def test_error_input_processor(self): + class TestItem(Item): + name = Field() + + class TestItemLoader(ItemLoader): + default_item_class = TestItem + name_in = MapCompose(float) + + il = TestItemLoader() + self.assertRaises(ValueError, il.add_value, 'name', + [u'marta', u'other']) + + def test_error_output_processor(self): + class TestItem(Item): + name = Field() + + class TestItemLoader(ItemLoader): + default_item_class = TestItem + name_out = Compose(Join(), float) + + il = TestItemLoader() + il.add_value('name', u'marta') + with self.assertRaises(ValueError): + il.load_item() + + def test_error_processor_as_argument(self): + class TestItem(Item): + name = Field() + + class TestItemLoader(ItemLoader): + default_item_class = TestItem + + il = TestItemLoader() + self.assertRaises(ValueError, il.add_value, 'name', + [u'marta', u'other'], Compose(float)) + + +class InitializationFromDictTest(unittest.TestCase): + + item_class = dict + + def test_keep_single_value(self): + """Loaded item should contain values from the initial item""" + input_item = self.item_class(name='foo') + il = ItemLoader(item=input_item) + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(dict(loaded_item), {'name': ['foo']}) + + def test_keep_list(self): + """Loaded item should contain values from the initial item""" + input_item = self.item_class(name=['foo', 'bar']) + il = ItemLoader(item=input_item) + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']}) + + def test_add_value_singlevalue_singlevalue(self): + """Values added after initialization should be appended""" + input_item = self.item_class(name='foo') + il = ItemLoader(item=input_item) + il.add_value('name', 'bar') + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']}) + + def test_add_value_singlevalue_list(self): + """Values added after initialization should be appended""" + input_item = self.item_class(name='foo') + il = ItemLoader(item=input_item) + il.add_value('name', ['item', 'loader']) + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']}) + + def test_add_value_list_singlevalue(self): + """Values added after initialization should be appended""" + input_item = self.item_class(name=['foo', 'bar']) + il = ItemLoader(item=input_item) + il.add_value('name', 'qwerty') + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']}) + + def test_add_value_list_list(self): + """Values added after initialization should be appended""" + input_item = self.item_class(name=['foo', 'bar']) + il = ItemLoader(item=input_item) + il.add_value('name', ['item', 'loader']) + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']}) + + def test_get_output_value_singlevalue(self): + """Getting output value must not remove value from item""" + input_item = self.item_class(name='foo') + il = ItemLoader(item=input_item) + self.assertEqual(il.get_output_value('name'), ['foo']) + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(loaded_item, dict({'name': ['foo']})) + + def test_get_output_value_list(self): + """Getting output value must not remove value from item""" + input_item = self.item_class(name=['foo', 'bar']) + il = ItemLoader(item=input_item) + self.assertEqual(il.get_output_value('name'), ['foo', 'bar']) + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']})) + + def test_values_single(self): + """Values from initial item must be added to loader._values""" + input_item = self.item_class(name='foo') + il = ItemLoader(item=input_item) + self.assertEqual(il._values.get('name'), ['foo']) + + def test_values_list(self): + """Values from initial item must be added to loader._values""" + input_item = self.item_class(name=['foo', 'bar']) + il = ItemLoader(item=input_item) + self.assertEqual(il._values.get('name'), ['foo', 'bar']) + + +class BaseNoInputReprocessingLoader(ItemLoader): + title_in = MapCompose(str.upper) + title_out = TakeFirst() + + +class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader): + default_item_class = dict + + +class NoInputReprocessingFromDictTest(unittest.TestCase): + """ + Loaders initialized from loaded items must not reprocess fields (dict instances) + """ + def test_avoid_reprocessing_with_initial_values_single(self): + il = NoInputReprocessingDictLoader(item=dict(title='foo')) + il_loaded = il.load_item() + self.assertEqual(il_loaded, dict(title='foo')) + self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo')) + + def test_avoid_reprocessing_with_initial_values_list(self): + il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar'])) + il_loaded = il.load_item() + self.assertEqual(il_loaded, dict(title='foo')) + self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo')) + + def test_avoid_reprocessing_without_initial_values_single(self): + il = NoInputReprocessingDictLoader() + il.add_value('title', 'foo') + il_loaded = il.load_item() + self.assertEqual(il_loaded, dict(title='FOO')) + self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO')) + + def test_avoid_reprocessing_without_initial_values_list(self): + il = NoInputReprocessingDictLoader() + il.add_value('title', ['foo', 'bar']) + il_loaded = il.load_item() + self.assertEqual(il_loaded, dict(title='FOO')) + self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO')) + + +class TestOutputProcessorDict(unittest.TestCase): + def test_output_processor(self): + + class TempDict(dict): + def __init__(self, *args, **kwargs): + super(TempDict, self).__init__(self, *args, **kwargs) + self.setdefault('temp', 0.3) + + class TempLoader(ItemLoader): + default_item_class = TempDict + default_input_processor = Identity() + default_output_processor = Compose(TakeFirst()) + + loader = TempLoader() + item = loader.load_item() + self.assertIsInstance(item, TempDict) + self.assertEqual(dict(item), {'temp': 0.3}) + + +class ProcessorsTest(unittest.TestCase): + + def test_take_first(self): + proc = TakeFirst() + self.assertEqual(proc([None, '', 'hello', 'world']), 'hello') + self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0) + + def test_identity(self): + proc = Identity() + self.assertEqual(proc([None, '', 'hello', 'world']), + [None, '', 'hello', 'world']) + + def test_join(self): + proc = Join() + self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) + self.assertEqual(proc(['', 'hello', 'world']), u' hello world') + self.assertEqual(proc(['hello', 'world']), u'hello world') + self.assertIsInstance(proc(['hello', 'world']), str) + + def test_compose(self): + proc = Compose(lambda v: v[0], str.upper) + self.assertEqual(proc(['hello', 'world']), 'HELLO') + proc = Compose(str.upper) + self.assertEqual(proc(None), None) + proc = Compose(str.upper, stop_on_none=False) + self.assertRaises(ValueError, proc, None) + proc = Compose(str.upper, lambda x: x + 1) + self.assertRaises(ValueError, proc, 'hello') + + def test_mapcompose(self): + def filter_world(x): + return None if x == 'world' else x + proc = MapCompose(filter_world, str.upper) + self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), + [u'HELLO', u'THIS', u'IS', u'SCRAPY']) + proc = MapCompose(filter_world, str.upper) + self.assertEqual(proc(None), []) + proc = MapCompose(filter_world, str.upper) + self.assertRaises(ValueError, proc, [1]) + proc = MapCompose(filter_world, lambda x: x + 1) + self.assertRaises(ValueError, proc, 'hello') + + +class SelectJmesTestCase(unittest.TestCase): + test_list_equals = { + 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), + 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), + 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'dict': ( + 'foo.bar[*].name', + {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, + ['one', 'two'] + ), + 'list': ('[1]', [1, 2], 2) + } + + def test_output(self): + for tl in self.test_list_equals: + expr, test_list, expected = self.test_list_equals[tl] + test = SelectJmes(expr)(test_list) + self.assertEqual( + test, + expected, + msg='test "{}" got {} expected {}'.format(tl, test, expected) + ) + + +# Functions as processors + +def function_processor_strip(iterable): + return [x.strip() for x in iterable] + + +def function_processor_upper(iterable): + return [x.upper() for x in iterable] + + +class FunctionProcessorItem(Item): + foo = Field( + input_processor=function_processor_strip, + output_processor=function_processor_upper, + ) + + +class FunctionProcessorDictLoader(ItemLoader): + default_item_class = dict + foo_in = function_processor_strip + foo_out = function_processor_upper + + +class FunctionProcessorTestCase(unittest.TestCase): + + def test_processor_defined_in_item_loader(self): + lo = FunctionProcessorDictLoader() + lo.add_value('foo', ' bar ') + lo.add_value('foo', [' asdf ', ' qwerty ']) + self.assertEqual( + dict(lo.load_item()), + {'foo': ['BAR', 'ASDF', 'QWERTY']} + ) + + +class DeprecatedUtilityFunctionsTestCase(unittest.TestCase): + + def test_deprecated_wrap_loader_context(self): + def function(*args): + return None + + with warnings.catch_warnings(record=True) as w: + wrap_loader_context(function, context=dict()) + + assert len(w) == 1 + assert issubclass(w[0].category, ScrapyDeprecationWarning) + + def test_deprecated_extract_regex(self): + with warnings.catch_warnings(record=True) as w: + extract_regex(r'\w+', 'this is a test') + + assert len(w) == 1 + assert issubclass(w[0].category, ScrapyDeprecationWarning) + + +if __name__ == "__main__": + unittest.main() From 8bdcdb0a76e3780681dcfbbd4a0eee62e2bb05b1 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 16 Jul 2020 09:13:54 +0300 Subject: [PATCH 181/568] Add quotes to example in docs --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0bb5f1733..7e91b365d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -461,7 +461,7 @@ For instance, if your settings include:: And your :command:`crawl` command line is:: - scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json + scrapy crawl spidername -o 'dirname/%(batch_id)d-filename%(batch_time)s.json' The command line above can generate a directory tree like:: From 0e0d1ad64323033017c4a01893b5337a980cb5ec Mon Sep 17 00:00:00 2001 From: Marc <noviluni@gmail.com> Date: Thu, 16 Jul 2020 14:19:46 +0200 Subject: [PATCH 182/568] remove python 2 reminiscence in cookies --- scrapy/http/cookies.py | 3 --- tests/test_http_cookies.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 3e810992c..0c97e6999 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -186,9 +186,6 @@ class WrappedResponse: def info(self): return self - # python3 cookiejars calls get_all def get_all(self, name, default=None): return [to_unicode(v, errors='replace') for v in self.response.headers.getlist(name)] - # python2 cookiejars calls getheaders - getheaders = get_all diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 45ddb42ba..540e27907 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -64,9 +64,6 @@ class WrappedResponseTest(TestCase): def test_info(self): self.assertIs(self.wrapped.info(), self.wrapped) - def test_getheaders(self): - self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html']) - def test_get_all(self): # get_all result must be native string self.assertEqual(self.wrapped.get_all('content-type'), ['text/html']) From b97a39fda0b72d8a4dcd631599e5d9bf530a5bee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com> Date: Thu, 16 Jul 2020 17:38:22 +0200 Subject: [PATCH 183/568] deprecate retry_on_eintr (#4683) --- scrapy/utils/python.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index afa8a8135..9204977cf 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -285,6 +285,7 @@ class WeakKeyCache: return self._weakdict[key] +@deprecated def retry_on_eintr(function, *args, **kw): """Run a function and retry it while getting EINTR errors""" while True: From 41263f61c6de8048023ba4c80e062f56b21e5a19 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 16 Jul 2020 18:41:45 +0300 Subject: [PATCH 184/568] Change single quotes to double in example in docs --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7e91b365d..fdc6e7cba 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -461,7 +461,7 @@ For instance, if your settings include:: And your :command:`crawl` command line is:: - scrapy crawl spidername -o 'dirname/%(batch_id)d-filename%(batch_time)s.json' + scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json" The command line above can generate a directory tree like:: From d29bec60d795b13ecb6e5978cb9e4d8fbd298b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 16 Jul 2020 23:19:24 +0200 Subject: [PATCH 185/568] Upgrade PyPy for CI, and test both 3.5 (oldest) and 3.6 (newest) (#4504) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Upgrade PyPy for CI, and test both 3.5 (oldest) and 3.6 (newest) * Log a detailed error message to discover why MockServer is not working * Go for all lines! * Disable tests based on mitmproxy while running on PyPy * Fix test_get_func_args for PyPy 3.6+ * Make testPayloadDefaultCiphers work regardless of OpenSSL default ciphers * Crossing fingers… * Rename: testPayloadDefaultCiphers → testPayloadDisabledCipher * Test the PyPy version currently documented as the minimum required version * Fix the PYPY_VERSION tag * Update the documentation about supported PyPy versions * Also test the latest 3.5 Python version with PyPy * Fix the PYPY_VERSION value for the latest 3.5 version * Use pinned dependencies for asyncio and PyPy tests against oldest supported Python versions * Fix PyPy installation for the pypy3-pinned Tox environment * Try installing Cython * Maybe PyPy requires lxml 3.6.0? * install.rst: minor clarification * lxml 4.0.0 is required on PyPy * Require setuptools 18.5+ * Revert "Require setuptools 18.5+" This reverts commit 017ec33ac2d237523cdd53be9be8169dd540759e. * Maintain lxml as a dependency if setuptools < 18.5 is used --- .travis.yml | 16 +++++++++----- docs/faq.rst | 14 ------------ docs/intro/install.rst | 12 +++++++--- setup.py | 44 +++++++++++++++++++++++-------------- tests/test_proxy_connect.py | 2 ++ tests/test_utils_python.py | 6 ++++- tests/test_webclient.py | 6 +++-- tox.ini | 31 +++++++++++++++++--------- 8 files changed, 80 insertions(+), 51 deletions(-) diff --git a/.travis.yml b/.travis.yml index b403ac54c..db720b918 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,19 +18,25 @@ matrix: - env: TOXENV=typing python: 3.8 - - env: TOXENV=pypy3 - env: TOXENV=pinned python: 3.5.2 - - env: TOXENV=asyncio + - env: TOXENV=asyncio-pinned python: 3.5.2 # We use additional code to support 3.5.3 and earlier + - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0 + - env: TOXENV=py python: 3.5 - env: TOXENV=asyncio python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 + - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 + - env: TOXENV=py python: 3.6 + - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 + - env: TOXENV=py python: 3.7 + - env: TOXENV=py PYPI_RELEASE_JOB=true python: 3.8 dist: bionic @@ -42,9 +48,9 @@ matrix: dist: bionic install: - | - if [ "$TOXENV" = "pypy3" ]; then - export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable" - wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" + if [[ ! -z "$PYPY_VERSION" ]]; then + export PYPY_VERSION="pypy$PYPY_VERSION-linux64" + wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2" tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" diff --git a/docs/faq.rst b/docs/faq.rst index d5ea3cb87..ea2c8216f 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -64,20 +64,6 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars .. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use -.. _faq-python-versions: - -What Python versions does Scrapy support? ------------------------------------------ - -Scrapy is supported under Python 3.5.2+ -under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). -Python 3 support was added in Scrapy 1.1. -PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5. -Python 2 support was dropped in Scrapy 2.0. - -.. note:: - For Python 3 support on Windows, it is recommended to use - Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`. Did Scrapy "steal" X from Django? --------------------------------- diff --git a/docs/intro/install.rst b/docs/intro/install.rst index fb64d443c..6d65ae2ee 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -4,12 +4,18 @@ Installation guide ================== +.. _faq-python-versions: + +Supported Python versions +========================= + +Scrapy requires Python 3.5.2+, either the CPython implementation (default) or +the PyPy 5.9+ implementation (see :ref:`python:implementations`). + + Installing Scrapy ================= -Scrapy runs on Python 3.5.2 or above under CPython (default Python -implementation) and PyPy (starting with PyPy 5.9). - If you're using `Anaconda`_ or `Miniconda`_, you can install the package from the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows and macOS. diff --git a/setup.py b/setup.py index f8d9b491b..58090f7a2 100644 --- a/setup.py +++ b/setup.py @@ -18,12 +18,39 @@ def has_environment_marker_platform_impl_support(): return parse_version(setuptools_version) >= parse_version('18.5') +install_requires = [ + 'Twisted>=17.9.0', + 'cryptography>=2.0', + 'cssselect>=0.9.1', + 'itemloaders>=1.0.1', + 'lxml>=3.5.0', + 'parsel>=1.5.0', + 'PyDispatcher>=2.0.5', + 'pyOpenSSL>=16.2.0', + 'queuelib>=1.4.2', + 'service_identity>=16.0.0', + 'w3lib>=1.17.0', + 'zope.interface>=4.1.3', + 'protego>=0.1.15', + 'itemadapter>=0.1.0', +] extras_require = {} if has_environment_marker_platform_impl_support(): + extras_require[':platform_python_implementation == "CPython"'] = [ + 'lxml>=3.5.0', + ] extras_require[':platform_python_implementation == "PyPy"'] = [ + # Earlier lxml versions are affected by + # https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no, + # which was fixed in Cython 0.26, released on 2017-06-19, and used to + # generate the C headers of lxml release tarballs published since then, the + # first of which was: + 'lxml>=4.0.0', 'PyPyDispatcher>=2.1.0', ] +else: + install_requires.append('lxml>=3.5.0') setup( @@ -67,21 +94,6 @@ setup( 'Topic :: Software Development :: Libraries :: Python Modules', ], python_requires='>=3.5.2', - install_requires=[ - 'Twisted>=17.9.0', - 'cryptography>=2.0', - 'cssselect>=0.9.1', - 'itemloaders>=1.0.1', - 'lxml>=3.5.0', - 'parsel>=1.5.0', - 'PyDispatcher>=2.0.5', - 'pyOpenSSL>=16.2.0', - 'queuelib>=1.4.2', - 'service_identity>=16.0.0', - 'w3lib>=1.17.0', - 'zope.interface>=4.1.3', - 'protego>=0.1.15', - 'itemadapter>=0.1.0', - ], + install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index fc5658ae7..a56e3c39a 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -60,6 +60,8 @@ def _wrong_credentials(proxy_url): @skipIf(sys.version_info < (3, 5, 4), "requires mitmproxy < 3.0.0, which these tests do not support") +@skipIf("pypy" in sys.executable, + "mitmproxy does not support PyPy") @skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), "mitmproxy does not support Windows when running Python < 3.7") class ProxyConnectTestCase(TestCase): diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 65e6ba876..ebce3c079 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -4,6 +4,7 @@ import operator import platform import unittest from itertools import count +from sys import version_info from warnings import catch_warnings from scrapy.utils.python import ( @@ -214,9 +215,12 @@ class UtilsPythonTestCase(unittest.TestCase): else: self.assertEqual( get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) - self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) self.assertEqual( get_func_args(operator.itemgetter(2), stripself=True), ['obj']) + if version_info < (3, 6): + self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) + else: + self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable']) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index c1c5945c2..ee64d455c 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -413,7 +413,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): self.getURL("payload"), body=s, contextFactory=client_context_factory ).addCallback(self.assertEqual, to_bytes(s)) - def testPayloadDefaultCiphers(self): + def testPayloadDisabledCipher(self): s = "0123456789" * 10 - d = getPage(self.getURL("payload"), body=s, contextFactory=ScrapyClientContextFactory()) + settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) + client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) + d = getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory) return self.assertFailure(d, OpenSSL.SSL.Error) diff --git a/tox.ini b/tox.ini index 5d79739bb..4557c63e3 100644 --- a/tox.ini +++ b/tox.ini @@ -58,11 +58,6 @@ deps = commands = pylint conftest.py docs extras scrapy setup.py tests -[testenv:pypy3] -basepython = pypy3 -commands = - py.test {posargs:--durations=10 docs scrapy tests} - [pinned] deps = -ctests/constraints.txt @@ -85,7 +80,6 @@ deps = Pillow==3.4.2 [testenv:pinned] -basepython = python3 deps = {[pinned]deps} lxml==3.5.0 @@ -104,6 +98,27 @@ deps = reppy robotexclusionrulesparser +[testenv:asyncio] +commands = + {[testenv]commands} --reactor=asyncio + +[testenv:asyncio-pinned] +commands = {[testenv:asyncio]commands} +deps = {[testenv:pinned]deps} + +[testenv:pypy3] +basepython = pypy3 +commands = + py.test {posargs:--durations=10 docs scrapy tests} + +[testenv:pypy3-pinned] +basepython = {[testenv:pypy3]basepython} +commands = {[testenv:pypy3]commands} +deps = + {[pinned]deps} + lxml==4.0.0 + PyPyDispatcher==2.1.0 + [docs] changedir = docs deps = @@ -135,7 +150,3 @@ deps = {[docs]deps} setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck - -[testenv:asyncio] -commands = - {[testenv]commands} --reactor=asyncio From 62a4ede5e995f83abd5a90f7dd6ac242f2f3870d Mon Sep 17 00:00:00 2001 From: Artur Shellunts <shellunts.artur@gmail.com> Date: Fri, 17 Jul 2020 12:40:04 +0200 Subject: [PATCH 186/568] Remove deprecated classes BaseSgmlLinkExtractor, RegexLinkExtractor and SgmlLinkExtractor (#4356) --- scrapy/linkextractors/regex.py | 41 ----- scrapy/linkextractors/sgml.py | 151 ------------------ tests/ignores.txt | 2 - .../link_extractor/linkextractor.html | 2 +- .../link_extractor/linkextractor_latin1.html | 2 +- 5 files changed, 2 insertions(+), 196 deletions(-) delete mode 100644 scrapy/linkextractors/regex.py delete mode 100644 scrapy/linkextractors/sgml.py diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py deleted file mode 100644 index 3f2557248..000000000 --- a/scrapy/linkextractors/regex.py +++ /dev/null @@ -1,41 +0,0 @@ -import re -from urllib.parse import urljoin - -from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url - -from scrapy.link import Link -from scrapy.linkextractors.sgml import SgmlLinkExtractor - - -linkre = re.compile( - "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>", - re.DOTALL | re.IGNORECASE) - - -def clean_link(link_text): - """Remove leading and trailing whitespace and punctuation""" - return link_text.strip("\t\r\n '\"\x0c") - - -class RegexLinkExtractor(SgmlLinkExtractor): - """High performant link extractor""" - - def _extract_links(self, response_text, response_url, response_encoding, base_url=None): - def clean_text(text): - return replace_escape_chars(remove_tags(text.decode(response_encoding))).strip() - - def clean_url(url): - clean_url = '' - try: - clean_url = urljoin(base_url, replace_entities(clean_link(url.decode(response_encoding)))) - except ValueError: - pass - return clean_url - - if base_url is None: - base_url = get_base_url(response_text, response_url, response_encoding) - - links_text = linkre.findall(response_text) - return [Link(clean_url(url).encode(response_encoding), - clean_text(text)) - for url, _, text in links_text] diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py deleted file mode 100644 index 2ba6bca45..000000000 --- a/scrapy/linkextractors/sgml.py +++ /dev/null @@ -1,151 +0,0 @@ -""" -SGMLParser-based Link extractors -""" -import warnings -from urllib.parse import urljoin -from sgmllib import SGMLParser - -from w3lib.url import safe_url_string, canonicalize_url -from w3lib.html import strip_html5_whitespace - -from scrapy.link import Link -from scrapy.linkextractors import FilteringLinkExtractor -from scrapy.utils.misc import arg_to_iter, rel_has_nofollow -from scrapy.utils.python import unique as unique_list, to_unicode -from scrapy.utils.response import get_base_url -from scrapy.exceptions import ScrapyDeprecationWarning - - -class BaseSgmlLinkExtractor(SGMLParser): - - def __init__(self, tag="a", attr="href", unique=False, process_value=None, - strip=True, canonicalized=False): - warnings.warn( - "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. " - "Please use scrapy.linkextractors.LinkExtractor", - ScrapyDeprecationWarning, stacklevel=2, - ) - SGMLParser.__init__(self) - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.process_value = (lambda v: v) if process_value is None else process_value - self.current_link = None - self.unique = unique - self.strip = strip - if canonicalized: - self.link_key = lambda link: link.url - else: - self.link_key = lambda link: canonicalize_url(link.url, - keep_fragments=True) - - def _extract_links(self, response_text, response_url, response_encoding, base_url=None): - """ Do the real extraction work """ - self.reset() - self.feed(response_text) - self.close() - - ret = [] - if base_url is None: - base_url = urljoin(response_url, self.base_url) if self.base_url else response_url - for link in self.links: - if isinstance(link.url, str): - link.url = link.url.encode(response_encoding) - try: - link.url = urljoin(base_url, link.url) - except ValueError: - continue - link.url = safe_url_string(link.url, response_encoding) - link.text = to_unicode(link.text, response_encoding, errors='replace').strip() - ret.append(link) - - return ret - - def _process_links(self, links): - """ Normalize and filter extracted links - - The subclass should override it if necessary - """ - return unique_list(links, key=self.link_key) if self.unique else links - - def extract_links(self, response): - # wrapper needed to allow to work directly with text - links = self._extract_links(response.body, response.url, response.encoding) - links = self._process_links(links) - return links - - def reset(self): - SGMLParser.reset(self) - self.links = [] - self.base_url = None - self.current_link = None - - def unknown_starttag(self, tag, attrs): - if tag == 'base': - self.base_url = dict(attrs).get('href') - if self.scan_tag(tag): - for attr, value in attrs: - if self.scan_attr(attr): - if self.strip and value is not None: - value = strip_html5_whitespace(value) - url = self.process_value(value) - if url is not None: - link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel'))) - self.links.append(link) - self.current_link = link - - def unknown_endtag(self, tag): - if self.scan_tag(tag): - self.current_link = None - - def handle_data(self, data): - if self.current_link: - self.current_link.text = self.current_link.text + data - - def matches(self, url): - """This extractor matches with any url, since - it doesn't contain any patterns""" - return True - - -class SgmlLinkExtractor(FilteringLinkExtractor): - - def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, - process_value=None, deny_extensions=None, restrict_css=(), - strip=True, restrict_text=()): - warnings.warn( - "SgmlLinkExtractor is deprecated and will be removed in future releases. " - "Please use scrapy.linkextractors.LinkExtractor", - ScrapyDeprecationWarning, stacklevel=2, - ) - - tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) - tag_func = lambda x: x in tags - attr_func = lambda x: x in attrs - - with warnings.catch_warnings(): - warnings.simplefilter('ignore', ScrapyDeprecationWarning) - lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process_value=process_value, strip=strip, - canonicalized=canonicalize) - - super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions, - restrict_text=restrict_text) - - def extract_links(self, response): - base_url = None - if self.restrict_xpaths: - base_url = get_base_url(response) - body = u''.join(f - for x in self.restrict_xpaths - for f in response.xpath(x).getall() - ).encode(response.encoding, errors='xmlcharrefreplace') - else: - body = response.body - - links = self._extract_links(body, response.url, response.encoding, base_url) - links = self._process_links(links) - return links diff --git a/tests/ignores.txt b/tests/ignores.txt index f6e0d6fbe..222288841 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -1,5 +1,3 @@ -scrapy/linkextractors/sgml.py -scrapy/linkextractors/regex.py scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py scrapy/extensions/memusage.py diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html index 7d5db368a..2307ea865 100644 --- a/tests/sample_data/link_extractor/linkextractor.html +++ b/tests/sample_data/link_extractor/linkextractor.html @@ -1,7 +1,7 @@ <html> <head> <base href='http://example.com' /> -<title>Sample page with links for testing RegexLinkExtractor +Sample page with links for testing LinkExtractor
diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html index fc31d7e5d..e7eee18de 100644 --- a/tests/sample_data/link_extractor/linkextractor_latin1.html +++ b/tests/sample_data/link_extractor/linkextractor_latin1.html @@ -2,7 +2,7 @@ - Sample page with links for testing RegexLinkExtractor + Sample page with links for testing LinkExtractor
From 86f7ac2f2b5d58e0b2588fa2aa4c777a8decf299 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Fri, 17 Jul 2020 17:48:25 +0300 Subject: [PATCH 187/568] Try to fix error at Windows --- tests/test_feedexport.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 129b7fc0b..cc124624d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1204,6 +1204,8 @@ class BatchDeliveriesTest(FeedExportTestBase): for path, feed in FEEDS.items(): dir_name = os.path.dirname(path) + if not os.path.exists(str(dir_name)): + continue for file in sorted(os.listdir(dir_name)): with open(os.path.join(dir_name, file), 'rb') as f: data = f.read() From 3e0492741d93b05c464457b3b128a2b0d24c994b Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Sun, 19 Jul 2020 00:10:29 +0300 Subject: [PATCH 188/568] Another try to fix test errors on Windows --- tests/test_feedexport.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index cc124624d..c49b2e92f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1190,9 +1190,14 @@ class BatchDeliveriesTest(FeedExportTestBase): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ + def build_url(path): + if path[0] != '/': + path = '/' + path + return urljoin('file:', path) + FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { - urljoin('file:', file_path): feed + build_url(file_path): feed for file_path, feed in FEEDS.items() } content = defaultdict(list) @@ -1204,8 +1209,6 @@ class BatchDeliveriesTest(FeedExportTestBase): for path, feed in FEEDS.items(): dir_name = os.path.dirname(path) - if not os.path.exists(str(dir_name)): - continue for file in sorted(os.listdir(dir_name)): with open(os.path.join(dir_name, file), 'rb') as f: data = f.read() From de297a3a167a6be3ac1ed94a891effefc12a6d00 Mon Sep 17 00:00:00 2001 From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com> Date: Mon, 20 Jul 2020 17:53:38 +0530 Subject: [PATCH 189/568] enable ANSI color (instead of ANSI color codes) in the Windows terminal #4393 (#4403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * changed ie. -> i.e.(spelling error) on lines 667, 763 (issue scrapy#4332) * updated all text files for issue #4332 (ie. -> i.e.) * Apply ie. → i.e. in source comments * ie → e.g. * modified scrapy/utils/display.py to stop ANSI color sequences in the Windows terminal (issue #4393) * modified scrapy/utils/display.py to stop ANSI color sequences in the Windows terminal (issue #4393) * enabled virtual terminal processing (pr #4403) * check for specific windows 10 version (pr #4403) * fixing flake-8 test (pr #4403) * added error handling for terminal info (pr #4403) * corrected stderr (pr #4403) * changed orientation, removed unwanted spaces (pr #4403) * no need for style variable (pr #4403) * fixing trailing whitespaces * commenting windows check * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * small fixes * Shifting _color_support_info() function * enabled virtual terminal processing (pr #4403) * check for specific windows 10 version (pr #4403) * fixing flake-8 test (pr #4403) * added error handling for terminal info (pr #4403) * corrected stderr (pr #4403) * changed orientation, removed unwanted spaces (pr #4403) * no need for style variable (pr #4403) * fixing trailing whitespaces * commenting windows check * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * small fixes * Shifting _color_support_info() function * error handling * error handlingy * raise ValueError * added in-built function for version comparison * recommit changes * changed check -> parse * version comparison -> parse_version * added scrapy/utils/display.py in pytest.ini * Trigger * Add simple test for scrapy.utils.display._colorize * Flake8: E501 for tests/test_utils_display.py * assertEquals -> assertEqual * Normal formatter for all platforms * separate test for windows * all curses under try block * added global TestStr * more test added * small fix * covering exceptions * windows test failing * Refactor output color handling * Fix pprint test * fix flake8 Co-authored-by: Adrián Chaves Co-authored-by: Eugenio Lacuesta --- scrapy/utils/display.py | 28 +++++++++++-- tests/test_utils_display.py | 78 +++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 tests/test_utils_display.py diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 9735220ef..f4d17224b 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -2,20 +2,42 @@ pprint and pformat wrappers with colorization support """ +import ctypes +import platform import sys +from distutils.version import LooseVersion as parse_version from pprint import pformat as pformat_ +def _enable_windows_terminal_processing(): + # https://stackoverflow.com/a/36760881 + kernel32 = ctypes.windll.kernel32 + return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7)) + + +def _tty_supports_color(): + if sys.platform != "win32": + return True + + if parse_version(platform.version()) < parse_version("10.0.14393"): + return True + + # Windows >= 10.0.14393 interprets ANSI escape sequences providing terminal + # processing is enabled. + return _enable_windows_terminal_processing() + + def _colorize(text, colorize=True): - if not colorize or not sys.stdout.isatty(): + if not colorize or not sys.stdout.isatty() or not _tty_supports_color(): return text try: from pygments import highlight + except ImportError: + return text + else: from pygments.formatters import TerminalFormatter from pygments.lexers import PythonLexer return highlight(text, PythonLexer(), TerminalFormatter()) - except ImportError: - return text def pformat(obj, *args, **kwargs): diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py new file mode 100644 index 000000000..9ec8311d9 --- /dev/null +++ b/tests/test_utils_display.py @@ -0,0 +1,78 @@ +from io import StringIO + +from unittest import mock, TestCase + +from scrapy.utils.display import pformat, pprint + + +class TestDisplay(TestCase): + object = {'a': 1} + colorized_string = ( + "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" + "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n" + ) + plain_string = "{'a': 1}" + + @mock.patch('sys.platform', 'linux') + @mock.patch("sys.stdout.isatty") + def test_pformat(self, isatty): + isatty.return_value = True + self.assertEqual(pformat(self.object), self.colorized_string) + + @mock.patch("sys.stdout.isatty") + def test_pformat_dont_colorize(self, isatty): + isatty.return_value = True + self.assertEqual(pformat(self.object, colorize=False), self.plain_string) + + def test_pformat_not_tty(self): + self.assertEqual(pformat(self.object), self.plain_string) + + @mock.patch('sys.platform', 'win32') + @mock.patch('platform.version') + @mock.patch("sys.stdout.isatty") + def test_pformat_old_windows(self, isatty, version): + isatty.return_value = True + version.return_value = '10.0.14392' + self.assertEqual(pformat(self.object), self.colorized_string) + + @mock.patch('sys.platform', 'win32') + @mock.patch('scrapy.utils.display._enable_windows_terminal_processing') + @mock.patch('platform.version') + @mock.patch("sys.stdout.isatty") + def test_pformat_windows_no_terminal_processing(self, isatty, version, terminal_processing): + isatty.return_value = True + version.return_value = '10.0.14393' + terminal_processing.return_value = False + self.assertEqual(pformat(self.object), self.plain_string) + + @mock.patch('sys.platform', 'win32') + @mock.patch('scrapy.utils.display._enable_windows_terminal_processing') + @mock.patch('platform.version') + @mock.patch("sys.stdout.isatty") + def test_pformat_windows(self, isatty, version, terminal_processing): + isatty.return_value = True + version.return_value = '10.0.14393' + terminal_processing.return_value = True + self.assertEqual(pformat(self.object), self.colorized_string) + + @mock.patch('sys.platform', 'linux') + @mock.patch("sys.stdout.isatty") + def test_pformat_no_pygments(self, isatty): + isatty.return_value = True + + import builtins + real_import = builtins.__import__ + + def mock_import(name, globals, locals, fromlist, level): + if 'pygments' in name: + raise ImportError + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = mock_import + self.assertEqual(pformat(self.object), self.plain_string) + builtins.__import__ = real_import + + def test_pprint(self): + with mock.patch('sys.stdout', new=StringIO()) as mock_out: + pprint(self.object) + self.assertEqual(mock_out.getvalue(), "{'a': 1}\n") From ece4fa6c7cf7fdce9e5e77f56b209941ad401f53 Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 7 Mar 2020 19:21:45 +0000 Subject: [PATCH 190/568] Fix ignored testcase: boto is never installed --- tests/test_feedexport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 37384081a..21da9fdcd 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -182,9 +182,9 @@ class S3FeedStorageTest(unittest.TestCase): create=True) def test_parse_credentials(self): try: - import boto # noqa: F401 + import botocore # noqa: F401 except ImportError: - raise unittest.SkipTest("S3FeedStorage requires boto") + raise unittest.SkipTest("S3FeedStorage requires botocore") aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} crawler = get_crawler(settings_dict=aws_credentials) From 234c8b8c50a2db0d6d71455c382957099f4c0dcc Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 7 Mar 2020 19:31:56 +0000 Subject: [PATCH 191/568] Removing deprecated S3FeedStorage without AWS keys instancing. --- scrapy/extensions/feedexport.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 72a34ae0d..e793c12dc 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -94,23 +94,10 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None, acl=None): - # BEGIN Backward compatibility for initialising without keys (and - # without using from_crawler) - no_defaults = access_key is None and secret_key is None - if no_defaults: - from scrapy.utils.project import get_project_settings - settings = get_project_settings() - if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings: - warnings.warn( - "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " - "without AWS keys is deprecated. Please supply credentials or " - "use the `from_crawler()` constructor.", - category=ScrapyDeprecationWarning, - stacklevel=2 - ) - access_key = settings['AWS_ACCESS_KEY_ID'] - secret_key = settings['AWS_SECRET_ACCESS_KEY'] - # END Backward compatibility + no_keys = access_key is None and secret_key is None + if no_keys: + raise NotConfigured('%s is missing AWS credentials' % + self.__class__.__name__) u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key From 98e8086d1b3da8906d1395f453b1299b4e3b499e Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 7 Mar 2020 19:21:09 +0000 Subject: [PATCH 192/568] Adapt S3FeedStorage testcase --- tests/test_feedexport.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 21da9fdcd..34a67fff3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -24,6 +24,7 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner +from scrapy.exceptions import NotConfigured from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, @@ -176,10 +177,6 @@ class BlockingFeedStorageTest(unittest.TestCase): class S3FeedStorageTest(unittest.TestCase): - @mock.patch('scrapy.utils.project.get_project_settings', - new=mock.MagicMock(return_value={'AWS_ACCESS_KEY_ID': 'conf_key', - 'AWS_SECRET_ACCESS_KEY': 'conf_secret'}), - create=True) def test_parse_credentials(self): try: import botocore # noqa: F401 @@ -205,12 +202,9 @@ class S3FeedStorageTest(unittest.TestCase): aws_credentials['AWS_SECRET_ACCESS_KEY']) self.assertEqual(storage.access_key, 'uri_key') self.assertEqual(storage.secret_key, 'uri_secret') - # Backward compatibility for initialising without settings - with warnings.catch_warnings(record=True) as w: - storage = S3FeedStorage('s3://mybucket/export.csv') - self.assertEqual(storage.access_key, 'conf_key') - self.assertEqual(storage.secret_key, 'conf_secret') - self.assertTrue('without AWS keys' in str(w[-1].message)) + # Instantiate without credentials + with self.assertRaises(NotConfigured): + S3FeedStorage('s3://mybucket/export.csv') @defer.inlineCallbacks def test_store(self): From 2829cd4268d22a328acd64a6816c479b1fe21f39 Mon Sep 17 00:00:00 2001 From: nyov Date: Tue, 17 Mar 2020 10:19:13 +0000 Subject: [PATCH 193/568] Allow use without credentials --- scrapy/extensions/feedexport.py | 4 ---- tests/test_feedexport.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e793c12dc..68d6533d3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -94,10 +94,6 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None, acl=None): - no_keys = access_key is None and secret_key is None - if no_keys: - raise NotConfigured('%s is missing AWS credentials' % - self.__class__.__name__) u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 34a67fff3..656bb515f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -24,7 +24,6 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner -from scrapy.exceptions import NotConfigured from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, @@ -202,9 +201,6 @@ class S3FeedStorageTest(unittest.TestCase): aws_credentials['AWS_SECRET_ACCESS_KEY']) self.assertEqual(storage.access_key, 'uri_key') self.assertEqual(storage.secret_key, 'uri_secret') - # Instantiate without credentials - with self.assertRaises(NotConfigured): - S3FeedStorage('s3://mybucket/export.csv') @defer.inlineCallbacks def test_store(self): From 430d22e46ebb67aba39005365e2e68d834d8fb86 Mon Sep 17 00:00:00 2001 From: Artur Shellunts Date: Tue, 21 Jul 2020 23:39:04 +0200 Subject: [PATCH 194/568] Remove not used import warnings --- tests/test_feedexport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 656bb515f..e80e07554 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,7 +5,6 @@ import random import shutil import string import tempfile -import warnings from io import BytesIO from logging import getLogger from pathlib import Path From 316620b517207b1082dd9c5b4ebfc7fbe745e3bb Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 22 Jul 2020 13:53:46 +0530 Subject: [PATCH 195/568] chore: pass spider as argument for request method - download_maxsize and download_warnsize can now be extracted from the spider directly and passed to the stream - remove `partial` flag from the response as per RFC 7540 - Section 8.1.2.6 --- scrapy/core/http2/protocol.py | 12 +++++--- scrapy/core/http2/stream.py | 35 ++++++++++------------ tests/test_http2_client_protocol.py | 46 +++++++++++++++++++---------- 3 files changed, 55 insertions(+), 38 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 041908116..feb034a0c 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -28,6 +28,7 @@ from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request from scrapy.settings import Settings +from scrapy.spiders import Spider logger = logging.getLogger(__name__) @@ -71,7 +72,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # ID of the next request stream # Following the convention - 'Streams initiated by a client MUST - # use odd-numbered stream identifiers' (RFC 7540) + # use odd-numbered stream identifiers' (RFC 7540 - Section 5.1.1) self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs @@ -136,6 +137,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._active_streams += 1 stream = self._pending_request_stream_pool.popleft() stream.initiate_request() + self._write_to_transport() def pop_stream(self, stream_id: int) -> Stream: """Perform cleanup when a stream is closed @@ -145,13 +147,15 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._send_pending_requests() return stream - def _new_stream(self, request: Request) -> Stream: + def _new_stream(self, request: Request, spider: Spider) -> Stream: """Instantiates a new Stream object """ stream = Stream( stream_id=next(self._stream_id_generator), request=request, protocol=self, + download_maxsize=getattr(spider, 'download_maxsize', self.metadata['default_download_maxsize']), + download_warnsize=getattr(spider, 'download_warnsize', self.metadata['default_download_warnsize']), ) self.streams[stream.stream_id] = stream return stream @@ -166,11 +170,11 @@ class H2ClientProtocol(Protocol, TimeoutMixin): data = self.conn.data_to_send() self.transport.write(data) - def request(self, request: Request) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred: if not isinstance(request, Request): raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}') - stream = self._new_stream(request) + stream = self._new_stream(request, spider) d = stream.get_response() # Add the stream to the request pool diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 15f081cab..133196797 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -83,7 +83,9 @@ class Stream: self, stream_id: int, request: Request, - protocol: "H2ClientProtocol" + protocol: "H2ClientProtocol", + download_maxsize: int = 0, + download_warnsize: int = 0 ) -> None: """ Arguments: @@ -95,14 +97,8 @@ class Stream: self._request: Request = request self._protocol: "H2ClientProtocol" = protocol - self._download_maxsize = self._request.meta.get( - 'download_maxsize', - self._protocol.metadata['default_download_maxsize'] - ) - self._download_warnsize = self._request.meta.get( - 'download_warnsize', - self._protocol.metadata['default_download_warnsize'] - ) + self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) + self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) self.request_start_time = None @@ -346,26 +342,28 @@ class Stream: self.stream_closed_server = True - flags = None - if b'Content-Length' not in self._response['headers']: - # Missing Content-Length - {twisted.web.http.PotentialDataLoss} - flags = ['partial'] + # We do not check for Content-Length or Transfer-Encoding in response headers + # and add `partial` flag as in HTTP/1.1 as 'A request or response that includes + # a payload body can include a content-length header field' (RFC 7540 - Section 8.1.2.6) # NOTE: Order of handling the events is important here # As we immediately cancel the request when maxsize is exceeded while # receiving DATA_FRAME's when we have received the headers (not # having Content-Length) if reason is StreamCloseReason.MAXSIZE_EXCEEDED: - expected_size = int(self._response['headers'].get(b'Content-Length', -1)) + expected_size = int(self._response['headers'].get( + b'Content-Length', + self._response['flow_controlled_size']) + ) error_msg = ( - f'Cancelling download of {self._request.url}: expected response ' - f'size ({expected_size}) larger than download max size ({self._download_maxsize}).' + f'Cancelling download of {self._request.url}: received response ' + f'size ({expected_size}) larger than download max size ({self._download_maxsize})' ) logger.error(error_msg) self._deferred_response.errback(CancelledError(error_msg)) elif reason is StreamCloseReason.ENDED: - self._fire_response_deferred(flags) + self._fire_response_deferred() # Stream was abruptly ended here elif reason is StreamCloseReason.CANCELLED: @@ -398,7 +396,7 @@ class Stream: f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' )) - def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None: + def _fire_response_deferred(self) -> None: """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" @@ -416,7 +414,6 @@ class Stream: headers=self._response['headers'], body=body, request=self._request, - flags=flags, certificate=self._protocol.metadata['certificate'], ip_address=self._protocol.metadata['ip_address'], ) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 2833801e7..d0386f7f8 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -23,6 +23,7 @@ from scrapy.core.http2.protocol import H2ClientFactory from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings +from scrapy.spiders import Spider from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -41,6 +42,14 @@ def make_html_body(val): return bytes(response, 'utf-8') +class DummySpider(Spider): + name = 'dummy' + start_urls = [] + + def parse(self, response): + print(response) + + class Data: SMALL_SIZE = 1024 # 1 KB LARGE_SIZE = 1024 ** 2 # 1 MB @@ -210,6 +219,9 @@ class Https2ClientProtocolTestCase(TestCase): assert len(path) > 0 and (path[0] == '/' or path[0] == '&') return f'{self.scheme}://{self.hostname}:{self.port_number}{path}' + def make_request(self, request: Request) -> Deferred: + return self.client.request(request, DummySpider()) + @staticmethod def _check_repeat(get_deferred, count): d_list = [] @@ -233,7 +245,7 @@ class Https2ClientProtocolTestCase(TestCase): content_length = int(response.headers.get('Content-Length')) self.assertEqual(len(response.body), content_length) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(check_response) d.addErrback(self.fail) return d @@ -273,7 +285,7 @@ class Https2ClientProtocolTestCase(TestCase): expected_extra_data, expected_status: int ): - d = self.client.request(request) + d = self.make_request(request) def assert_response(response: Response): self.assertEqual(response.status, expected_status) @@ -355,7 +367,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, 499) self.assertEqual(response.request, request) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_response) d.addErrback(self.fail) d.cancel() @@ -367,8 +379,13 @@ class Https2ClientProtocolTestCase(TestCase): def assert_cancelled_error(failure): self.assertIsInstance(failure.value, CancelledError) + error_pattern = re.compile( + rf'Cancelling download of {request.url}: received response ' + rf'size \(\d*\) larger than download max size \(1000\)' + ) + self.assertEqual(len(re.findall(error_pattern, str(failure.value))), 1) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(self.fail) d.addErrback(assert_cancelled_error) return d @@ -385,7 +402,7 @@ class Https2ClientProtocolTestCase(TestCase): for error in failure.value.reasons )) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(self.fail) d.addErrback(assert_failure) return d @@ -397,10 +414,9 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, 200) self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) self.assertEqual(response.request, request) - self.assertIn('partial', response.flags) self.assertNotIn('Content-Length', response.headers) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_content_length) d.addErrback(self.fail) return d @@ -413,7 +429,7 @@ class Https2ClientProtocolTestCase(TestCase): expected_body ): with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm: - response = yield self.client.request(request) + response = yield self.make_request(request) self.assertEqual(response.status, 200) self.assertEqual(response.request, request) self.assertEqual(response.body, expected_body) @@ -469,13 +485,13 @@ class Https2ClientProtocolTestCase(TestCase): # Send 100 request (we do not check the result) for _ in range(100): - d = self.client.request(Request(self.get_url('/get-data-html-small'))) + d = self.make_request(Request(self.get_url('/get-data-html-small'))) d.addBoth(lambda _: None) d_list.append(d) # Now send 10 extra request and save the response deferred in a list for _ in range(10): - d = self.client.request(Request(self.get_url('/get-data-html-small'))) + d = self.make_request(Request(self.get_url('/get-data-html-small'))) d.addCallback(self.fail) d.addErrback(assert_inactive_stream) d_list.append(d) @@ -488,7 +504,7 @@ class Https2ClientProtocolTestCase(TestCase): def test_invalid_request_type(self): with self.assertRaises(TypeError): - self.client.request('https://InvalidDataTypePassed.com') + self.make_request('https://InvalidDataTypePassed.com') def test_query_parameters(self): params = { @@ -504,7 +520,7 @@ class Https2ClientProtocolTestCase(TestCase): data = json.loads(str(response.body, content_encoding)) self.assertEqual(data, params) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_query_params) d.addErrback(self.fail) @@ -517,7 +533,7 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [] for status in [200, 404]: request = Request(self.get_url(f'/status?n={status}')) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_response_status, status) d.addErrback(self.fail) d_list.append(d) @@ -537,7 +553,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertIsInstance(response.ip_address, IPv4Address) self.assertEqual(str(response.ip_address), '127.0.0.1') - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_metadata) d.addErrback(self.fail) @@ -553,7 +569,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertIn('127.0.0.1', error_msg) self.assertIn(str(request), error_msg) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(self.fail) d.addErrback(assert_invalid_hostname) return d From 3685e99cca47a5006258cd4246f255216797641f Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 22 Jul 2020 14:47:20 +0530 Subject: [PATCH 196/568] test: http2 connection timeout --- tests/test_http2_client_protocol.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index d0386f7f8..746eef4d6 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -18,8 +18,9 @@ from twisted.web.client import URI from twisted.web.http import Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File +from twisted.internet.error import TimeoutError -from scrapy.core.http2.protocol import H2ClientFactory +from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings @@ -134,6 +135,11 @@ class NoContentLengthHeader(LeafResource): request.finish() +class TimeoutResponse(LeafResource): + def render_GET(self, request: TxRequest): + return NOT_DONE_YET + + class QueryParams(LeafResource): def render_GET(self, request: TxRequest): request.setHeader('Content-Type', 'application/json; charset=UTF-8') @@ -172,6 +178,7 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'no-content-length-header', NoContentLengthHeader()) r.putChild(b'status', Status()) r.putChild(b'query-params', QueryParams()) + r.putChild(b'timeout', TimeoutResponse()) return r @inlineCallbacks @@ -590,3 +597,22 @@ class Https2ClientProtocolTestCase(TestCase): ] return DeferredList(d_list, fireOnOneErrback=True) + + def test_connection_timeout(self): + request = Request(self.get_url('/timeout')) + d = self.make_request(request) + + # Update the timer to 1s to test connection timeout + self.client.setTimeout(1) + + def assert_timeout_error(failure: Failure): + for err in failure.value.reasons: + if isinstance(err, TimeoutError): + self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err)) + break + else: + self.fail() + + d.addCallback(self.fail) + d.addErrback(assert_timeout_error) + return d From 9fffb801ed3dedbb3935c811c7d61ed953ff22dc Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 8 Jul 2020 20:18:38 +0530 Subject: [PATCH 197/568] feat: H2Agent, H2ConnectionPool base implementation --- scrapy/core/downloader/handlers/http2.py | 55 ++++++++++++++++++++ scrapy/core/http2/agent.py | 64 ++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 scrapy/core/downloader/handlers/http2.py create mode 100644 scrapy/core/http2/agent.py diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py new file mode 100644 index 000000000..0dc06f4d8 --- /dev/null +++ b/scrapy/core/downloader/handlers/http2.py @@ -0,0 +1,55 @@ +import warnings + +from scrapy.core.downloader.tls import openssl_methods +from scrapy.core.http2.agent import H2Agent, H2ConnectionPool +from scrapy.http.request import Request +from scrapy.settings import Settings +from scrapy.utils.misc import create_instance, load_object + + +class H2DownloadHandler: + def __init__(self, settings: Settings, crawler=None): + self._crawler = crawler + + from twisted.internet import reactor + self._pool = H2ConnectionPool(reactor, settings) + + self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] + self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + # try method-aware context factory + try: + self._context_factory = create_instance( + objcls=self._context_factory_cls, + settings=settings, + crawler=crawler, + method=self._ssl_method, + ) + except TypeError: + # use context factory defaults + self._context_factory = create_instance( + objcls=self._context_factory_cls, + settings=settings, + crawler=crawler, + ) + msg = """ + '%s' does not accept `method` argument (type OpenSSL.SSL method,\ + e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" % ( + settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + warnings.warn(msg) + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler.settings, crawler) + + def download_request(self, request: Request, spider): + from twisted.internet import reactor + + agent = H2Agent(reactor, self._pool, self._context_factory) + d = agent.request(request) + + def print_result(result): + print(result) + + d.addCallback(print_result) + return d diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py new file mode 100644 index 000000000..e14e5d633 --- /dev/null +++ b/scrapy/core/http2/agent.py @@ -0,0 +1,64 @@ +from typing import Dict, Tuple + +from twisted.internet import defer +from twisted.internet.base import ReactorBase +from twisted.internet.defer import Deferred +from twisted.internet.endpoints import SSL4ClientEndpoint, optionsForClientTLS +from twisted.web.client import URI, BrowserLikePolicyForHTTPS + +from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory +from scrapy.http.request import Request +from scrapy.settings import Settings +from scrapy.utils.python import to_bytes, to_unicode + + +class H2ConnectionPool: + def __init__(self, reactor: ReactorBase, settings: Settings) -> None: + self._reactor = reactor + self.settings = settings + self._connections: Dict[Tuple, H2ClientProtocol] = {} + + def get_connection(self, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: + key = (uri.scheme, uri.host, uri.port) + conn = self._connections.get(key, None) + if conn: + return defer.succeed(conn) + return self._new_connection(key, uri, endpoint) + + def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: + factory = H2ClientFactory(uri, self.settings) + d = endpoint.connect(factory) + + def put_connection(conn: H2ClientProtocol) -> H2ClientProtocol: + self._connections[key] = conn + return conn + + d.addCallback(put_connection) + return d + + def _remove_connection(self, key) -> None: + conn = self._connections.pop(key) + conn.loseConnection() + + +class H2Agent: + def __init__( + self, reactor: ReactorBase, pool: H2ConnectionPool, + context_factory=BrowserLikePolicyForHTTPS() + ) -> None: + self._reactor = reactor + self._pool = pool + self._context_factory = context_factory + + def request(self, request: Request) -> Deferred: + uri = URI.fromBytes(to_bytes(request.url, encoding='ascii')) + # options = optionsForClientTLS(hostname=to_unicode(uri.host), acceptableProtocols=[b'h2']) + # Hacky fix: Use options instead of self._context_factory to make endpoint work for HTTP/2 + endpoint = SSL4ClientEndpoint(self._reactor, to_unicode(uri.host), uri.port, self._context_factory) + d = self._pool.get_connection(uri, endpoint) + + def cb_connected(conn: H2ClientProtocol): + return conn.request(request) + + d.addCallback(cb_connected) + return d From 8252a6f8d8930ce23bdf8a6f51038b4ce49d1968 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 15 Jul 2020 04:58:59 +0530 Subject: [PATCH 198/568] fix: H2Agent not able to connect via SSL - add H2WrappedContextFactory class which wraps the context factory passed to H2Agent and updates the SSL context acceptable protocols list to only h2 --- scrapy/core/downloader/handlers/http2.py | 1 + scrapy/core/http2/agent.py | 66 +++++++++++++++--------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 0dc06f4d8..c8b401e80 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -50,6 +50,7 @@ class H2DownloadHandler: def print_result(result): print(result) + return result d.addCallback(print_result) return d diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index e14e5d633..566c074c2 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,15 +1,19 @@ -from typing import Dict, Tuple +from typing import Dict, Tuple, Optional from twisted.internet import defer +from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred -from twisted.internet.endpoints import SSL4ClientEndpoint, optionsForClientTLS -from twisted.web.client import URI, BrowserLikePolicyForHTTPS +from twisted.internet.endpoints import SSL4ClientEndpoint +from twisted.python.failure import Failure +from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory +from twisted.web.iweb import IPolicyForHTTPS +from zope.interface import implementer +from zope.interface.verify import verifyObject from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory from scrapy.http.request import Request from scrapy.settings import Settings -from scrapy.utils.python import to_bytes, to_unicode class H2ConnectionPool: @@ -26,39 +30,51 @@ class H2ConnectionPool: return self._new_connection(key, uri, endpoint) def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: - factory = H2ClientFactory(uri, self.settings) + conn_lost_deferred = Deferred() + conn_lost_deferred.addCallback(self._remove_connection, key) + + factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) d = endpoint.connect(factory) - - def put_connection(conn: H2ClientProtocol) -> H2ClientProtocol: - self._connections[key] = conn - return conn - - d.addCallback(put_connection) + d.addCallback(self.put_connection, key) return d - def _remove_connection(self, key) -> None: - conn = self._connections.pop(key) - conn.loseConnection() + def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol: + self._connections[key] = conn + return conn + + def _remove_connection(self, reason: Failure, key: Tuple) -> None: + self._connections.pop(key) + + +@implementer(IPolicyForHTTPS) +class H2WrappedContextFactory: + def __init__(self, context_factory) -> None: + verifyObject(IPolicyForHTTPS, context_factory) + self._wrapped_context_factory = context_factory + + def creatorForNetloc(self, hostname, port) -> ClientTLSOptions: + options = self._wrapped_context_factory.creatorForNetloc(hostname, port) + _setAcceptableProtocols(options._ctx, [b'h2']) + return options class H2Agent: def __init__( self, reactor: ReactorBase, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS() + context_factory=BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None ) -> None: self._reactor = reactor self._pool = pool - self._context_factory = context_factory + self._context_factory = H2WrappedContextFactory(context_factory) + self._endpoint_factory = _StandardEndpointFactory( + self._reactor, self._context_factory, + connect_timeout, bind_address + ) def request(self, request: Request) -> Deferred: - uri = URI.fromBytes(to_bytes(request.url, encoding='ascii')) - # options = optionsForClientTLS(hostname=to_unicode(uri.host), acceptableProtocols=[b'h2']) - # Hacky fix: Use options instead of self._context_factory to make endpoint work for HTTP/2 - endpoint = SSL4ClientEndpoint(self._reactor, to_unicode(uri.host), uri.port, self._context_factory) + uri = URI.fromBytes(bytes(request.url, encoding='utf-8')) + endpoint = self._endpoint_factory.endpointForURI(uri) d = self._pool.get_connection(uri, endpoint) - - def cb_connected(conn: H2ClientProtocol): - return conn.request(request) - - d.addCallback(cb_connected) + d.addCallback(lambda conn: conn.request(request)) return d From 62ce842afc8ef829ffd6f164712a8a9413eb9e1d Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 15 Jul 2020 07:50:53 +0530 Subject: [PATCH 199/568] fix: multiple h2 connections to same uri - When multiple requests are sent to H2ConnectionPool to the same uri while the connection is in connecting state -- multiple connections were establised. - Fixed the bug using a deque of all the request deferred's which fire with the H2ClientProtocol (connection) instance when connection is established --- scrapy/core/http2/agent.py | 55 ++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 566c074c2..7a8847c38 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,11 +1,11 @@ -from typing import Dict, Tuple, Optional +from collections import deque +from typing import Deque, Dict, List, Tuple, Optional from twisted.internet import defer from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred -from twisted.internet.endpoints import SSL4ClientEndpoint -from twisted.python.failure import Failure +from twisted.internet.endpoints import HostnameEndpoint from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory from twisted.web.iweb import IPolicyForHTTPS from zope.interface import implementer @@ -20,31 +20,70 @@ class H2ConnectionPool: def __init__(self, reactor: ReactorBase, settings: Settings) -> None: self._reactor = reactor self.settings = settings + + # Store a dictionary which is used to get the respective + # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port) self._connections: Dict[Tuple, H2ClientProtocol] = {} - def get_connection(self, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: + # Save all requests that arrive before the connection is established + self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} + + def get_connection(self, uri: URI, endpoint: HostnameEndpoint) -> Deferred: key = (uri.scheme, uri.host, uri.port) + if key in self._pending_requests: + # Received a request while connecting to remote + # Create a deferred which will fire with the H2ClientProtocol + # instance + d = Deferred() + self._pending_requests[key].append(d) + return d + + # Check if we already have a connection to the remote conn = self._connections.get(key, None) if conn: + # Return this connection instance wrapped inside a deferred return defer.succeed(conn) + + # No connection is established for the given URI return self._new_connection(key, uri, endpoint) - def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: + def _new_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred: + self._pending_requests[key] = deque() + conn_lost_deferred = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) - d = endpoint.connect(factory) - d.addCallback(self.put_connection, key) + conn_d = endpoint.connect(factory) + conn_d.addCallback(self.put_connection, key) + + d = Deferred() + self._pending_requests[key].append(d) return d def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol: self._connections[key] = conn + + # Now as we have established a proper HTTP/2 connection + # we fire all the deferred's with the connection instance + pending_requests = self._pending_requests.pop(key) + while pending_requests: + d = pending_requests.popleft() + d.callback(conn) + + del pending_requests + return conn - def _remove_connection(self, reason: Failure, key: Tuple) -> None: + def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None: self._connections.pop(key) + # Call the errback of all the pending requests for this connection + pending_requests = self._pending_requests.pop(key, None) + while pending_requests: + d = pending_requests.popleft() + d.errback(errors) + @implementer(IPolicyForHTTPS) class H2WrappedContextFactory: From 031bfc9c3bed6c4dbfa4b7fcd48a8fc15f3582fb Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 22 Jul 2020 15:01:59 +0530 Subject: [PATCH 200/568] feat(wip): ScrapyH2Agent, ScrapyProxyH2Agent --- scrapy/core/downloader/contextfactory.py | 32 +++++ scrapy/core/downloader/handlers/http11.py | 27 +--- scrapy/core/downloader/handlers/http2.py | 161 +++++++++++++++++----- scrapy/core/http2/agent.py | 35 ++++- 4 files changed, 190 insertions(+), 65 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 452242d47..c0463cfc7 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,8 +1,12 @@ from OpenSSL import SSL +import warnings + from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer +from scrapy.core.downloader.tls import openssl_methods +from scrapy.utils.misc import create_instance, load_object from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS @@ -92,3 +96,31 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): trustRoot=platformTrust(), extraCertificateOptions={'method': self._ssl_method}, ) + + +def load_context_factory_from_settings(settings, crawler): + ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] + context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + # try method-aware context factory + try: + context_factory = create_instance( + objcls=context_factory_cls, + settings=settings, + crawler=crawler, + method=ssl_method, + ) + except TypeError: + # use context factory defaults + context_factory = create_instance( + objcls=context_factory_cls, + settings=settings, + crawler=crawler, + ) + msg = """ + '%s' does not accept `method` argument (type OpenSSL.SSL method,\ + e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" % ( + settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + warnings.warn(msg) + + return context_factory diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 22c9ac520..dac97ad29 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from zope.interface import implementer from scrapy import signals -from scrapy.core.downloader.tls import openssl_methods +from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload from scrapy.http import Headers from scrapy.responsetypes import responsetypes -from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import to_bytes, to_unicode @@ -43,29 +42,7 @@ class HTTP11DownloadHandler: self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False - self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] - self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - # try method-aware context factory - try: - self._contextFactory = create_instance( - objcls=self._contextFactoryClass, - settings=settings, - crawler=crawler, - method=self._sslMethod, - ) - except TypeError: - # use context factory defaults - self._contextFactory = create_instance( - objcls=self._contextFactoryClass, - settings=settings, - crawler=crawler, - ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) - warnings.warn(msg) + self._contextFactory = load_context_factory_from_settings(settings, crawler) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index c8b401e80..81ea78e69 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,10 +1,19 @@ import warnings +from time import time +from typing import Optional, Tuple +from urllib.parse import urldefrag -from scrapy.core.downloader.tls import openssl_methods +from twisted.internet.base import ReactorBase +from twisted.internet.error import TimeoutError +from twisted.web.client import URI + +from scrapy.core.downloader.contextfactory import load_context_factory_from_settings +from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool -from scrapy.http.request import Request +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.http import Request, Response from scrapy.settings import Settings -from scrapy.utils.misc import create_instance, load_object +from scrapy.spiders import Spider class H2DownloadHandler: @@ -13,44 +22,128 @@ class H2DownloadHandler: from twisted.internet import reactor self._pool = H2ConnectionPool(reactor, settings) - - self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] - self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - # try method-aware context factory - try: - self._context_factory = create_instance( - objcls=self._context_factory_cls, - settings=settings, - crawler=crawler, - method=self._ssl_method, - ) - except TypeError: - # use context factory defaults - self._context_factory = create_instance( - objcls=self._context_factory_cls, - settings=settings, - crawler=crawler, - ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) - warnings.warn(msg) + self._context_factory = load_context_factory_from_settings(settings, crawler) + self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') + self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') + self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') @classmethod def from_crawler(cls, crawler): return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider): + def download_request(self, request: Request, spider: Spider): + agent = ScrapyH2Agent( + context_factory=self._context_factory, + pool=self._pool, + maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), + warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), + crawler=self._crawler + ) + return agent.download_request(request, spider) + + def close(self) -> None: + self._pool.close_connections() + + +class ScrapyProxyH2Agent(H2Agent): + def __init__( + self, reactor: ReactorBase, + proxy_uri: URI, pool: H2ConnectionPool, + connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + ) -> None: + super(ScrapyProxyH2Agent, self).__init__( + reactor=reactor, + pool=pool, + connect_timeout=connect_timeout, + bind_address=bind_address + ) + self._proxy_uri = proxy_uri + + @staticmethod + def get_key(uri: URI) -> Tuple: + return "http-proxy", uri.host, uri.port + + +class ScrapyH2Agent: + _Agent = H2Agent + _ProxyAgent = ScrapyProxyH2Agent + + def __init__( + self, context_factory, + connect_timeout=10, + bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None, + maxsize: int = 0, warnsize: int = 0, + crawler=None + ) -> None: + self._context_factory = context_factory + self._connect_timeout = connect_timeout + self._bind_address = bind_address + self._pool = pool + self._maxsize = maxsize + self._warnsize = warnsize + self._crawler = crawler + + def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent: from twisted.internet import reactor + bind_address = request.meta.get('bindaddress') or self._bind_address + proxy = request.meta.get('proxy') + if proxy: + _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) + scheme = _parse(request.url)[0] + proxy_host = str(proxy_host, 'utf-8') + omit_connect_timeout = b'noconnect' in proxy_params + if omit_connect_timeout: + warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " + "If you use Crawlera, it doesn't require this mode anymore, " + "so you should update scrapy-crawlera to 1.3.0+ " + "and remove '?noconnect' from the Crawlera URL.", + ScrapyDeprecationWarning) - agent = H2Agent(reactor, self._pool, self._context_factory) - d = agent.request(request) + if scheme == b'https' and not omit_connect_timeout: + proxy_auth = request.headers.get(b'Proxy-Authorization', None) + proxy_conf = (proxy_host, proxy_port, proxy_auth) - def print_result(result): - print(result) - return result + # TODO: Return TunnelingAgent instance + else: + return self._ProxyAgent( + reactor=reactor, + proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')), + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool + ) - d.addCallback(print_result) + return self._Agent( + reactor=reactor, + context_factory=self._context_factory, + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool + ) + + def download_request(self, request: Request, spider: Spider): + from twisted.internet import reactor + timeout = request.meta.get('download_timeout') or self._connect_timeout + agent = self._get_agent(request, timeout) + + start_time = time() + d = agent.request(request, spider) + d.addCallback(self._cb_latency, request, start_time) + + timeout_cl = reactor.callLater(timeout, d.cancel) + d.addBoth(self._cb_timeout, request, timeout, timeout_cl) return d + + @staticmethod + def _cb_latency(response: Response, request: Request, start_time: float): + request.meta['download_latency'] = time() - start_time + return response + + @staticmethod + def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl): + if timeout_cl.active(): + timeout_cl.cancel() + return response + + url = urldefrag(request.url)[0] + raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 7a8847c38..c7a49fd42 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -6,7 +6,9 @@ from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOption from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint +from twisted.python.failure import Failure from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory +from twisted.web.error import SchemeNotSupported from twisted.web.iweb import IPolicyForHTTPS from zope.interface import implementer from zope.interface.verify import verifyObject @@ -14,6 +16,7 @@ from zope.interface.verify import verifyObject from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory from scrapy.http.request import Request from scrapy.settings import Settings +from scrapy.spiders import Spider class H2ConnectionPool: @@ -28,8 +31,7 @@ class H2ConnectionPool: # Save all requests that arrive before the connection is established self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} - def get_connection(self, uri: URI, endpoint: HostnameEndpoint) -> Deferred: - key = (uri.scheme, uri.host, uri.port) + def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred: if key in self._pending_requests: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol @@ -84,6 +86,15 @@ class H2ConnectionPool: d = pending_requests.popleft() d.errback(errors) + def close_connections(self) -> None: + """Close all the HTTP/2 connections and remove them from pool + + Returns: + Deferred that fires when all connections have been closed + """ + for conn in self._connections.values(): + conn.transport.loseConnection() + @implementer(IPolicyForHTTPS) class H2WrappedContextFactory: @@ -111,9 +122,21 @@ class H2Agent: connect_timeout, bind_address ) - def request(self, request: Request) -> Deferred: + def _get_endpoint(self, uri: URI): + return self._endpoint_factory.endpointForURI(uri) + + @staticmethod + def get_key(uri: URI) -> Tuple: + return uri.scheme, uri.host, uri.port + + def request(self, request: Request, spider: Spider) -> Deferred: uri = URI.fromBytes(bytes(request.url, encoding='utf-8')) - endpoint = self._endpoint_factory.endpointForURI(uri) - d = self._pool.get_connection(uri, endpoint) - d.addCallback(lambda conn: conn.request(request)) + try: + endpoint = self._get_endpoint(uri) + except SchemeNotSupported: + return defer.fail(Failure()) + + key = self.get_key(uri) + d = self._pool.get_connection(key, uri, endpoint) + d.addCallback(lambda conn: conn.request(request, spider)) return d From 8fae3d5bb768ebb3f9674cc38172226f404fb297 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 22 Jul 2020 16:08:35 -0300 Subject: [PATCH 201/568] Remove monkeypatches module from mypy section in setup.cfg --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 46a3d13fc..f8e7c0c91 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,9 +13,6 @@ follow_imports = skip [mypy-scrapy] ignore_errors = True -[mypy-scrapy._monkeypatches] -ignore_errors = True - [mypy-scrapy.commands] ignore_errors = True From a6c1d79b7cc3bc2c408eab356bbbf99a0536f110 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Tue, 28 Jul 2020 11:53:05 +0300 Subject: [PATCH 202/568] pep8 tiny changes --- docs/topics/feed-exports.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2c9774b55..dd4eb3c61 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -453,6 +453,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter FEED_EXPORT_BATCH_ITEM_COUNT ----------------------------- + Default: ``0`` If assigned an integer number higher than ``0``, Scrapy generates multiple output files @@ -474,7 +475,7 @@ generated: For instance, if your settings include:: - FEED_EXPORT_BATCH_ITEM_COUNT=100 + FEED_EXPORT_BATCH_ITEM_COUNT = 100 And your :command:`crawl` command line is:: From 52658539370c442e63102a3208781335953cdf53 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 28 Jul 2020 06:15:14 -0300 Subject: [PATCH 203/568] Use ItemAdapter.field_names when writing header in CsvItemExporter (#4668) --- scrapy/exporters.py | 8 +- tests/test_exporters.py | 203 ++++++++++++++++++++++++++++------------ 2 files changed, 146 insertions(+), 65 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 712572673..0aba1c904 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -243,12 +243,8 @@ class CsvItemExporter(BaseItemExporter): def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: if not self.fields_to_export: - if isinstance(item, dict): - # for dicts try using fields of the first item - self.fields_to_export = list(item.keys()) - else: - # use fields declared in Item - self.fields_to_export = list(item.fields.keys()) + # use declared field names, or keys if the item is a dict + self.fields_to_export = ItemAdapter(item).field_names() row = list(self._build_row(self.fields_to_export)) self.csv_writer.writerow(row) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index b27380309..25da54a65 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -8,6 +8,7 @@ from io import BytesIO from datetime import datetime import lxml.etree +from itemadapter import ItemAdapter from scrapy.item import Item, Field from scrapy.utils.python import to_unicode @@ -23,10 +24,37 @@ class TestItem(Item): age = Field() +def custom_serializer(value): + return str(int(value) + 2) + + +class CustomFieldItem(Item): + name = Field() + age = Field(serializer=custom_serializer) + + +try: + from dataclasses import make_dataclass, field +except ImportError: + TestDataClass = None + CustomFieldDataclass = None +else: + TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)]) + CustomFieldDataclass = make_dataclass( + "CustomFieldDataclass", + [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))] + ) + + class BaseItemExporterTest(unittest.TestCase): + item_class = TestItem + custom_field_item_class = CustomFieldItem + def setUp(self): - self.i = TestItem(name=u'John\xa3', age=u'22') + if self.item_class is None: + raise unittest.SkipTest("item class is None") + self.i = self.item_class(name=u'John\xa3', age=u'22') self.output = BytesIO() self.ie = self._get_exporter() @@ -39,7 +67,7 @@ class BaseItemExporterTest(unittest.TestCase): def _assert_expected_item(self, exported_dict): for k, v in exported_dict.items(): exported_dict[k] = to_unicode(v) - self.assertEqual(self.i, exported_dict) + self.assertEqual(self.i, self.item_class(**exported_dict)) def _get_nonstring_types_item(self): return { @@ -63,13 +91,14 @@ class BaseItemExporterTest(unittest.TestCase): self.assertItemExportWorks(self.i) def test_export_dict_item(self): - self.assertItemExportWorks(dict(self.i)) + self.assertItemExportWorks(ItemAdapter(self.i).asdict()) def test_serialize_field(self): - res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name']) + a = ItemAdapter(self.i) + res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name']) self.assertEqual(res, u'John\xa3') - res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']) + res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age']) self.assertEqual(res, u'22') def test_fields_to_export(self): @@ -82,18 +111,16 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(name, u'John\xa3') def test_field_custom_serializer(self): - def custom_serializer(value): - return str(int(value) + 2) - - class CustomFieldItem(Item): - name = Field() - age = Field(serializer=custom_serializer) - - i = CustomFieldItem(name=u'John\xa3', age=u'22') - + i = self.custom_field_item_class(name=u'John\xa3', age=u'22') + a = ItemAdapter(i) ie = self._get_exporter() - self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3') - self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') + self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3') + self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24') + + +class BaseItemExporterDataclassTest(BaseItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass class PythonItemExporterTest(BaseItemExporterTest): @@ -105,9 +132,9 @@ class PythonItemExporterTest(BaseItemExporterTest): PythonItemExporter(invalid_option='something') def test_nested_item(self): - i1 = TestItem(name=u'Joseph', age='22') + i1 = self.item_class(name=u'Joseph', age='22') i2 = dict(name=u'Maria', age=i1) - i3 = TestItem(name=u'Jesus', age=i2) + i3 = self.item_class(name=u'Jesus', age=i2) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual(type(exported), dict) @@ -119,9 +146,9 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age']['age']), dict) def test_export_list(self): - i1 = TestItem(name=u'Joseph', age='22') - i2 = TestItem(name=u'Maria', age=[i1]) - i3 = TestItem(name=u'Jesus', age=[i2]) + i1 = self.item_class(name=u'Joseph', age='22') + i2 = self.item_class(name=u'Maria', age=[i1]) + i3 = self.item_class(name=u'Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( @@ -132,9 +159,9 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_item_dict_list(self): - i1 = TestItem(name=u'Joseph', age='22') + i1 = self.item_class(name=u'Joseph', age='22') i2 = dict(name=u'Maria', age=[i1]) - i3 = TestItem(name=u'Jesus', age=[i2]) + i3 = self.item_class(name=u'Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( @@ -146,7 +173,7 @@ class PythonItemExporterTest(BaseItemExporterTest): def test_export_binary(self): exporter = PythonItemExporter(binary=True) - value = TestItem(name=u'John\xa3', age=u'22') + value = self.item_class(name=u'John\xa3', age=u'22') expected = {b'name': b'John\xc2\xa3', b'age': b'22'} self.assertEqual(expected, exporter.export_item(value)) @@ -157,6 +184,11 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(exported, item) +class PythonItemExporterDataclassTest(PythonItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class PprintItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -166,6 +198,11 @@ class PprintItemExporterTest(BaseItemExporterTest): self._assert_expected_item(eval(self.output.getvalue())) +class PprintItemExporterDataclassTest(PprintItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class PickleItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -175,8 +212,8 @@ class PickleItemExporterTest(BaseItemExporterTest): self._assert_expected_item(pickle.loads(self.output.getvalue())) def test_export_multiple_items(self): - i1 = TestItem(name='hello', age='world') - i2 = TestItem(name='bye', age='world') + i1 = self.item_class(name='hello', age='world') + i2 = self.item_class(name='bye', age='world') f = BytesIO() ie = PickleItemExporter(f) ie.start_exporting() @@ -184,8 +221,8 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.export_item(i2) ie.finish_exporting() f.seek(0) - self.assertEqual(pickle.load(f), i1) - self.assertEqual(pickle.load(f), i2) + self.assertEqual(self.item_class(**pickle.load(f)), i1) + self.assertEqual(self.item_class(**pickle.load(f)), i2) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -197,6 +234,11 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.loads(fp.getvalue()), item) +class PickleItemExporterDataclassTest(PickleItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class MarshalItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -219,6 +261,11 @@ class MarshalItemExporterTest(BaseItemExporterTest): self.assertEqual(marshal.load(fp), item) +class MarshalItemExporterDataclassTest(MarshalItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) @@ -245,18 +292,18 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_header_export_all(self): self.assertExportResult( item=self.i, - fields_to_export=self.i.fields.keys(), + fields_to_export=ItemAdapter(self.i).field_names(), expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_all_dict(self): self.assertExportResult( - item=dict(self.i), + item=ItemAdapter(self.i).asdict(), expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_single_field(self): - for item in [self.i, dict(self.i)]: + for item in [self.i, ItemAdapter(self.i).asdict()]: self.assertExportResult( item=item, fields_to_export=['age'], @@ -264,7 +311,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ) def test_header_export_two_items(self): - for item in [self.i, dict(self.i)]: + for item in [self.i, ItemAdapter(self.i).asdict()]: output = BytesIO() ie = CsvItemExporter(output) ie.start_exporting() @@ -275,7 +322,7 @@ class CsvItemExporterTest(BaseItemExporterTest): b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') def test_header_no_header_line(self): - for item in [self.i, dict(self.i)]: + for item in [self.i, ItemAdapter(self.i).asdict()]: self.assertExportResult( item=item, include_headers_line=False, @@ -309,6 +356,11 @@ class CsvItemExporterTest(BaseItemExporterTest): ) +class CsvItemExporterDataclassTest(CsvItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class XmlItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -318,8 +370,7 @@ class XmlItemExporterTest(BaseItemExporterTest): def xmltuple(elem): children = list(elem.iterchildren()) if children: - return [(child.tag, sorted(xmltuple(child))) - for child in children] + return [(child.tag, sorted(xmltuple(child))) for child in children] else: return [(elem.tag, [(elem.text, ())])] @@ -345,17 +396,21 @@ class XmlItemExporterTest(BaseItemExporterTest): def test_multivalued_fields(self): self.assertExportResult( - TestItem(name=[u'John\xa3', u'Doe']), - ( - b'\n' - b'John\xc2\xa3Doe' - ) + self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]), + b"""\n + + + John\xc2\xa3Doe + 123 + + + """ ) def test_nested_item(self): - i1 = TestItem(name=u'foo\xa3hoo', age='22') + i1 = dict(name=u'foo\xa3hoo', age='22') i2 = dict(name=u'bar', age=i1) - i3 = TestItem(name=u'buz', age=i2) + i3 = self.item_class(name=u'buz', age=i2) self.assertExportResult( i3, @@ -376,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_list_item(self): - i1 = TestItem(name=u'foo') + i1 = dict(name=u'foo') i2 = dict(name=u'bar', v2={"egg": ["spam"]}) - i3 = TestItem(name=u'buz', age=[i1, i2]) + i3 = self.item_class(name=u'buz', age=[i1, i2]) self.assertExportResult( i3, @@ -412,6 +467,12 @@ class XmlItemExporterTest(BaseItemExporterTest): ) +class XmlItemExporterDataclassTest(XmlItemExporterTest): + + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class JsonLinesItemExporterTest(BaseItemExporterTest): _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}} @@ -421,12 +482,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, dict(self.i)) + self.assertEqual(exported, ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = TestItem(name=u'Joseph', age='22') + i1 = self.item_class(name=u'Joseph', age='22') i2 = dict(name=u'Maria', age=i1) - i3 = TestItem(name=u'Jesus', age=i2) + i3 = self.item_class(name=u'Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() @@ -449,6 +510,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.assertEqual(exported, item) +class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest): + + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class JsonItemExporterTest(JsonLinesItemExporterTest): _expected_nested = [JsonLinesItemExporterTest._expected_nested] @@ -458,7 +525,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, [dict(self.i)]) + self.assertEqual(exported, [ItemAdapter(self.i).asdict()]) def assertTwoItemsExported(self, item): self.ie.start_exporting() @@ -466,28 +533,28 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual(exported, [dict(item), dict(item)]) + self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()]) def test_two_items(self): self.assertTwoItemsExported(self.i) def test_two_dict_items(self): - self.assertTwoItemsExported(dict(self.i)) + self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = TestItem(name=u'Joseph\xa3', age='22') - i2 = TestItem(name=u'Maria', age=i1) - i3 = TestItem(name=u'Jesus', age=i2) + i1 = self.item_class(name=u'Joseph\xa3', age='22') + i2 = self.item_class(name=u'Maria', age=i1) + i3 = self.item_class(name=u'Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}} + expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} self.assertEqual(exported, [expected]) def test_nested_dict_item(self): i1 = dict(name=u'Joseph\xa3', age='22') - i2 = TestItem(name=u'Maria', age=i1) + i2 = self.item_class(name=u'Maria', age=i1) i3 = dict(name=u'Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) @@ -506,7 +573,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertEqual(exported, [item]) -class CustomItemExporterTest(unittest.TestCase): +class JsonItemExporterDataclassTest(JsonItemExporterTest): + + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + +class CustomExporterItemTest(unittest.TestCase): + + item_class = TestItem + + def setUp(self): + if self.item_class is None: + raise unittest.SkipTest("item class is None") def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): @@ -516,16 +595,22 @@ class CustomItemExporterTest(unittest.TestCase): else: return super(CustomItemExporter, self).serialize_field(field, name, value) - i = TestItem(name=u'John', age='22') + i = self.item_class(name=u'John', age='22') + a = ItemAdapter(i) ie = CustomItemExporter() - self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John') - self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '23') + self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John') + self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23') i2 = {'name': u'John', 'age': '22'} self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John') self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23') +class CustomExporterDataclassTest(CustomExporterItemTest): + + item_class = TestDataClass + + if __name__ == '__main__': unittest.main() From e7a58fe1573176415a9ca054428c53c1ca29931a Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 29 Jul 2020 10:16:18 +0530 Subject: [PATCH 204/568] Code cleanup scrapy.utils.python.WeakKeyCache #4684 --- scrapy/utils/python.py | 12 ------------ tests/test_utils_python.py | 18 +----------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 9204977cf..7a393925e 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -273,18 +273,6 @@ def equal_attributes(obj1, obj2, attributes): return True -class WeakKeyCache: - - def __init__(self, default_factory): - self.default_factory = default_factory - self._weakdict = weakref.WeakKeyDictionary() - - def __getitem__(self, key): - if key not in self._weakdict: - self._weakdict[key] = self.default_factory(key) - return self._weakdict[key] - - @deprecated def retry_on_eintr(function, *args, **kw): """Run a function and retry it while getting EINTR errors""" diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index ebce3c079..b23ae2e52 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -9,7 +9,7 @@ from warnings import catch_warnings from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - WeakKeyCache, get_func_args, to_bytes, to_unicode, + get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) @@ -155,22 +155,6 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) - def test_weakkeycache(self): - class _Weakme: - pass - - _values = count() - wk = WeakKeyCache(lambda k: next(_values)) - k = _Weakme() - v = wk[k] - self.assertEqual(v, wk[k]) - self.assertNotEqual(v, wk[_Weakme()]) - self.assertEqual(v, wk[k]) - del k - for _ in range(100): - if wk._weakdict: - gc.collect() - self.assertFalse(len(wk._weakdict)) def test_get_func_args(self): def f1(a, b, c): From 92bec38591fccb523c2e643aef70a1f6cd7267ea Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 29 Jul 2020 13:43:59 +0530 Subject: [PATCH 205/568] feat: MethodNotAllowed405, Content-Length header - add tests to check for Content-Length header - raise MethodNotAllowed405 when remote send 'HTTP/2.0 405 Method Not Allowed' --- scrapy/core/downloader/handlers/http2.py | 15 +++----- scrapy/core/http2/agent.py | 2 +- scrapy/core/http2/protocol.py | 32 +++++++++++++---- scrapy/core/http2/stream.py | 14 ++++++-- tests/test_http2_client_protocol.py | 46 ++++++++++++++++++++++-- 5 files changed, 85 insertions(+), 24 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 81ea78e69..e9cc5ebbc 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -3,6 +3,7 @@ from time import time from typing import Optional, Tuple from urllib.parse import urldefrag +from twisted.internet.defer import Deferred from twisted.internet.base import ReactorBase from twisted.internet.error import TimeoutError from twisted.web.client import URI @@ -23,9 +24,6 @@ class H2DownloadHandler: from twisted.internet import reactor self._pool = H2ConnectionPool(reactor, settings) self._context_factory = load_context_factory_from_settings(settings, crawler) - self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') - self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') - self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') @classmethod def from_crawler(cls, crawler): @@ -35,8 +33,6 @@ class H2DownloadHandler: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, - maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), - warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), crawler=self._crawler ) return agent.download_request(request, spider) @@ -72,15 +68,12 @@ class ScrapyH2Agent: self, context_factory, connect_timeout=10, bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None, - maxsize: int = 0, warnsize: int = 0, crawler=None ) -> None: self._context_factory = context_factory self._connect_timeout = connect_timeout self._bind_address = bind_address self._pool = pool - self._maxsize = maxsize - self._warnsize = warnsize self._crawler = crawler def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent: @@ -121,7 +114,7 @@ class ScrapyH2Agent: pool=self._pool ) - def download_request(self, request: Request, spider: Spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: from twisted.internet import reactor timeout = request.meta.get('download_timeout') or self._connect_timeout agent = self._get_agent(request, timeout) @@ -135,12 +128,12 @@ class ScrapyH2Agent: return d @staticmethod - def _cb_latency(response: Response, request: Request, start_time: float): + def _cb_latency(response: Response, request: Request, start_time: float) -> Response: request.meta['download_latency'] = time() - start_time return response @staticmethod - def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl): + def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl) -> Response: if timeout_cl.active(): timeout_cl.cancel() return response diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index c7a49fd42..e62eef263 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -93,7 +93,7 @@ class H2ConnectionPool: Deferred that fires when all connections have been closed """ for conn in self._connections.values(): - conn.transport.loseConnection() + conn.transport.abortConnection() @implementer(IPolicyForHTTPS) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index feb034a0c..1ce8b6548 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -13,7 +13,7 @@ from h2.events import ( SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived, WindowUpdated ) -from h2.exceptions import H2Error, ProtocolError +from h2.exceptions import H2Error from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory @@ -39,7 +39,7 @@ class InvalidNegotiatedProtocol(H2Error): self.negotiated_protocol = negotiated_protocol def __str__(self) -> str: - return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}' + return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}' class RemoteTerminatedConnection(H2Error): @@ -48,7 +48,15 @@ class RemoteTerminatedConnection(H2Error): self.terminate_event = event def __str__(self) -> str: - return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address}' + return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address!r}' + + +class MethodNotAllowed405(H2Error): + def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]): + self.remote_ip_address = remote_ip_address + + def __str__(self) -> str: + return f"MethodNotAllowed405: Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}" @implementer(IHandshakeListener) @@ -217,14 +225,25 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # So, no need to send a GOAWAY frame to the remote self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) + def _check_received_data(self, data: bytes) -> None: + """Checks for edge cases where the connection to remote fails + without raising an appropriate H2Error + + Arguments: + data -- Data received from the remote + """ + if data.startswith(b'HTTP/2.0 405 Method Not Allowed'): + raise MethodNotAllowed405(self.metadata['ip_address']) + def dataReceived(self, data: bytes) -> None: # Reset the idle timeout as connection is still actively receiving data self.resetTimeout() try: + self._check_received_data(data) events = self.conn.receive_data(data) self._handle_events(events) - except ProtocolError as e: + except H2Error as e: # Save this error as ultimately the connection will be dropped # internally by hyper-h2. Saved error will be passed to all the streams # closed with the connection. @@ -271,9 +290,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, self._conn_lost_errors, from_protocol=True) + close_reason = StreamCloseReason.CONNECTION_LOST else: - stream.close(StreamCloseReason.INACTIVE, from_protocol=True) + close_reason = StreamCloseReason.INACTIVE + stream.close(close_reason, self._conn_lost_errors, from_protocol=True) self._active_streams -= len(self.streams) self.streams.clear() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 133196797..5bffa67e7 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -189,12 +189,15 @@ class Stream: headers = [ (':method', self._request.method), (':authority', url.netloc), - (':scheme', 'https'), + (':scheme', self._protocol.metadata['uri'].scheme), (':path', path), ] for name, value in self._request.headers.items(): - headers.append((name, value[0])) + headers.append((str(name, 'utf-8'), str(value[0], 'utf-8'))) + + if b'Content-Length' not in self._request.headers.keys(): + headers.append(('Content-Length', str(len(self._request.body)))) return headers @@ -337,6 +340,10 @@ class Stream: if not isinstance(reason, StreamCloseReason): raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}') + # Have default value of errors as an empty list as + # some cases can add a list of exceptions + errors = errors or [] + if not from_protocol: self._protocol.pop_stream(self.stream_id) @@ -387,7 +394,8 @@ class Stream: self._deferred_response.errback(ResponseFailed(errors)) elif reason is StreamCloseReason.INACTIVE: - self._deferred_response.errback(InactiveStreamClosed(self._request)) + errors.insert(0, InactiveStreamClosed(self._request)) + self._deferred_response.errback(ResponseFailed(errors)) elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 746eef4d6..4926ada14 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -11,14 +11,14 @@ from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint +from twisted.internet.error import TimeoutError from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase -from twisted.web.client import URI +from twisted.web.client import ResponseFailed, URI from twisted.web.http import Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File -from twisted.internet.error import TimeoutError from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname @@ -152,6 +152,19 @@ class QueryParams(LeafResource): return bytes(json.dumps(query_params), 'utf-8') +class RequestHeaders(LeafResource): + """Sends all the headers received as a response""" + + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') + headers = {} + for k, v in request.requestHeaders.getAllRawHeaders(): + headers[str(k, 'utf-8')] = str(v[0], 'utf-8') + + return bytes(json.dumps(headers), 'utf-8') + + def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -179,6 +192,7 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'status', Status()) r.putChild(b'query-params', QueryParams()) r.putChild(b'timeout', TimeoutResponse()) + r.putChild(b'request-headers', RequestHeaders()) return r @inlineCallbacks @@ -488,7 +502,11 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [] def assert_inactive_stream(failure): - self.assertIsNotNone(failure.check(InactiveStreamClosed)) + self.assertIsNotNone(failure.check(ResponseFailed)) + self.assertTrue(any( + isinstance(e, InactiveStreamClosed) + for e in failure.value.reasons + )) # Send 100 request (we do not check the result) for _ in range(100): @@ -616,3 +634,25 @@ class Https2ClientProtocolTestCase(TestCase): d.addCallback(self.fail) d.addErrback(assert_timeout_error) return d + + def test_request_headers_received(self): + request = Request(self.get_url('/request-headers'), headers={ + 'header-1': 'header value 1', + 'header-2': 'header value 2' + }) + d = self.make_request(request) + + def assert_request_headers(response: Response): + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + + response_headers = json.loads(str(response.body, 'utf-8')) + self.assertIsInstance(response_headers, dict) + for k, v in request.headers.items(): + k, v = str(k, 'utf-8'), str(v[0], 'utf-8') + self.assertIn(k, response_headers) + self.assertEqual(v, response_headers[k]) + + d.addErrback(self.fail) + d.addCallback(assert_request_headers) + return d From e8342996f6273b6b60ffc26a67e90dec34e96dfd Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 29 Jul 2020 13:51:01 +0530 Subject: [PATCH 206/568] test: H2DownloadHandler Following tests are skipped as Content-Length header not matching the data received is considered as a ProtocolError - test_download_broken_content_cause_data_loss - test_download_broken_chunked_content_cause_data_loss - test_download_broken_content_allow_data_loss - test_download_broken_chunked_content_allow_data_loss - test_download_broken_content_allow_data_loss_via_setting - test_download_broken_chunked_content_allow_data_loss_via_setting BREAKING CHANGES The following tests currently fail - test_content_length_zero_bodyless_post_request_headers - test_host_header_seted_in_request_headers - test_download_with_maxsize_very_large_file --- tests/test_downloader_handlers.py | 108 +++++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 9 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 51deb20f4..f6add82dc 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -23,8 +23,8 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler +from scrapy.core.downloader.handlers.http2 import H2DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler - from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Headers, Request from scrapy.http.response.text import TextResponse @@ -33,7 +33,6 @@ from scrapy.spiders import Spider from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto - from tests.mockserver import MockServer, ssl_context_factory, Echo from tests.spiders import SingleRequestSpider @@ -131,6 +130,7 @@ class ContentLengthHeaderResource(resource.Resource): A testing resource which renders itself as the value of the Content-Length header from the request. """ + def render(self, request): return request.requestHeaders.getRawHeaders(b"content-length")[0] @@ -142,6 +142,7 @@ class ChunkedResource(resource.Resource): request.write(b"chunked ") request.write(b"content\n") request.finish() + reactor.callLater(0, response) return server.NOT_DONE_YET @@ -155,6 +156,7 @@ class BrokenChunkedResource(resource.Resource): # Disable terminating chunk on finish. request.chunked = False closeConnection(request) + reactor.callLater(0, response) return server.NOT_DONE_YET @@ -186,6 +188,7 @@ class EmptyContentTypeHeaderResource(resource.Resource): A testing resource which renders itself as the value of request body without content-type header in response. """ + def render(self, request): request.setHeader("content-type", "") return request.content.read() @@ -197,12 +200,12 @@ class LargeChunkedFileResource(resource.Resource): for i in range(1024): request.write(b"x" * 1024) request.finish() + reactor.callLater(0, response) return server.NOT_DONE_YET class HttpTestCase(unittest.TestCase): - scheme = 'http' download_handler_cls = HTTPDownloadHandler @@ -230,10 +233,12 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) - self.host = 'localhost' + self.host = u'localhost' if self.scheme == 'https': + # Using WrappingFactory do not enable HTTP/2 failing all the + # tests with H2DownloadHandler self.port = reactor.listenSSL( - 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile), + 0, self.site, ssl_context_factory(self.keyfile, self.certfile), interface=self.host) else: self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) @@ -330,6 +335,7 @@ class HttpTestCase(unittest.TestCase): https://github.com/kennethreitz/requests/issues/405 https://bugs.python.org/issue14721 """ + def _test(response): self.assertEqual(response.body, b'0') @@ -514,6 +520,30 @@ class Https11TestCase(Http11TestCase): yield download_handler.close() +class Https2TestCase(Https11TestCase): + scheme = 'https' + download_handler_cls = H2DownloadHandler + HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + + def test_download_broken_content_cause_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_cause_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + class Https11WrongHostnameTestCase(Http11TestCase): scheme = 'https' @@ -526,6 +556,23 @@ class Https11WrongHostnameTestCase(Http11TestCase): certfile = 'keys/example-com.cert.pem' +class Https2WrongHostnameTestCase(Https2TestCase): + tls_log_message = ( + 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' + 'subject "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' + ) + + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = 'keys/example-com.key.pem' + certfile = 'keys/example-com.cert.pem' + + class Https11InvalidDNSId(Https11TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" @@ -534,6 +581,14 @@ class Https11InvalidDNSId(Https11TestCase): self.host = '127.0.0.1' +class Https2InvalidDNSId(Https2TestCase): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super(Https2InvalidDNSId, self).setUp() + self.host = '127.0.0.1' + + class Https11InvalidDNSPattern(Https11TestCase): """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" @@ -552,6 +607,24 @@ class Https11InvalidDNSPattern(Https11TestCase): super(Https11InvalidDNSPattern, self).setUp() +class Https2InvalidDNSPattern(Https2TestCase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = 'keys/localhost.ip.key' + certfile = 'keys/localhost.ip.crt' + + def setUp(self): + try: + from service_identity.exceptions import CertificateError # noqa: F401 + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") + self.tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' + 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + ) + super(Https2InvalidDNSPattern, self).setUp() + + class Https11CustomCiphers(unittest.TestCase): scheme = 'https' download_handler_cls = HTTP11DownloadHandler @@ -565,10 +638,9 @@ class Https11CustomCiphers(unittest.TestCase): FilePath(self.tmpname).child("file").setContent(b"0123456789") r = static.File(self.tmpname) self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) self.host = 'localhost' self.port = reactor.listenSSL( - 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'), + 0, self.site, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'), interface=self.host) self.portno = self.port.getHost().port crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}) @@ -593,6 +665,11 @@ class Https11CustomCiphers(unittest.TestCase): return d +class Https2CustomCiphers(Https11CustomCiphers): + scheme = 'https' + download_handler_cls = H2DownloadHandler + + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" @@ -644,6 +721,10 @@ class Http11MockServerTestCase(unittest.TestCase): self.assertTrue(reason, 'finished') +class Http2MockServerTestCase(Http11MockServerTestCase): + """HTTP 2.0 test case with MockServer""" + + class UriResource(resource.Resource): """Return the full uri that was requested""" @@ -734,6 +815,11 @@ class Http11ProxyTestCase(HttpProxyTestCase): self.assertIn(domain, timeout.osError) +# TODO: +class Http2ProxyTestCase(Http11ProxyTestCase): + download_handler_cls = H2DownloadHandler + + class HttpDownloadHandlerMock: def __init__(self, *args, **kwargs): @@ -931,7 +1017,6 @@ class S3TestCase(unittest.TestCase): class BaseFTPTestCase(unittest.TestCase): - username = "scrapy" password = "passwd" req_meta = {"ftp_user": username, "ftp_password": password} @@ -969,6 +1054,7 @@ class BaseFTPTestCase(unittest.TestCase): def _clean(data): self.download_handler.client.transport.loseConnection() return data + deferred.addCallback(_clean) if callback: deferred.addCallback(callback) @@ -985,6 +1071,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'I have the power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) + return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): @@ -998,6 +1085,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'Moooooooooo power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']}) + return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): @@ -1007,6 +1095,7 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.status, 404) + return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): @@ -1027,6 +1116,7 @@ class BaseFTPTestCase(unittest.TestCase): with open(local_fname, "rb") as f: self.assertEqual(f.read(), b"I have the power!") os.remove(local_fname) + return self._add_test_callbacks(d, _test) @@ -1043,11 +1133,11 @@ class FTPTestCase(BaseFTPTestCase): def _test(r): self.assertEqual(r.type, ConnectionLost) + return self._add_test_callbacks(d, errback=_test) class AnonymousFTPTestCase(BaseFTPTestCase): - username = "anonymous" req_meta = {} From 19f2b4b53dd51044083a9749f00366f41ed795c7 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 29 Jul 2020 17:25:59 +0530 Subject: [PATCH 207/568] refactor: AcceptableProtocolsContextFactory - rename H2WrappedContextFactory to AcceptableProtocolsContextFactory - AcceptableProtocolsContextFactory accepts an argument acceptable_protocols which can be used to override the context factory priority list of protocols during ALPN or NPN --- scrapy/core/downloader/handlers/http2.py | 2 +- scrapy/core/http2/agent.py | 9 +++++---- scrapy/core/http2/stream.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index e9cc5ebbc..411e06a78 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -3,8 +3,8 @@ from time import time from typing import Optional, Tuple from urllib.parse import urldefrag -from twisted.internet.defer import Deferred from twisted.internet.base import ReactorBase +from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index e62eef263..aa51508a5 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -97,14 +97,15 @@ class H2ConnectionPool: @implementer(IPolicyForHTTPS) -class H2WrappedContextFactory: - def __init__(self, context_factory) -> None: +class AcceptableProtocolsContextFactory: + def __init__(self, context_factory, acceptable_protocols: List[bytes]) -> None: verifyObject(IPolicyForHTTPS, context_factory) self._wrapped_context_factory = context_factory + self._acceptable_protocols = acceptable_protocols def creatorForNetloc(self, hostname, port) -> ClientTLSOptions: options = self._wrapped_context_factory.creatorForNetloc(hostname, port) - _setAcceptableProtocols(options._ctx, [b'h2']) + _setAcceptableProtocols(options._ctx, self._acceptable_protocols) return options @@ -116,7 +117,7 @@ class H2Agent: ) -> None: self._reactor = reactor self._pool = pool - self._context_factory = H2WrappedContextFactory(context_factory) + self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2']) self._endpoint_factory = _StandardEndpointFactory( self._reactor, self._context_factory, connect_timeout, bind_address diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 5bffa67e7..acdd46320 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -85,7 +85,7 @@ class Stream: request: Request, protocol: "H2ClientProtocol", download_maxsize: int = 0, - download_warnsize: int = 0 + download_warnsize: int = 0, ) -> None: """ Arguments: From 403bc7020a5e1ba2b59eced2cc5f4453c7650666 Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 29 Jul 2020 18:05:33 +0530 Subject: [PATCH 208/568] Code cleanup scrapy.utils.python.WeakKeyCache #4684 and fixing ci alerts --- tests/test_utils_python.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index b23ae2e52..5a53d89e4 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,9 +1,7 @@ import functools -import gc import operator import platform import unittest -from itertools import count from sys import version_info from warnings import catch_warnings @@ -12,7 +10,6 @@ from scrapy.utils.python import ( get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) - __doctests__ = ['scrapy.utils.python'] @@ -155,7 +152,6 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) - def test_get_func_args(self): def f1(a, b, c): pass From 49337bd2ae094d97d364948569f59b8211c8dbbe Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Thu, 30 Jul 2020 12:25:21 +0530 Subject: [PATCH 210/568] Code cleanup scrapy.utils.python.WeakKeyCache #4684 and fixing ci alerts --- scrapy/utils/python.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7a393925e..c8f921ff3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -127,6 +127,7 @@ def re_rsearch(pattern, text, chunk_size=1024): In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing the start position of the match, and the ending (regarding the entire text). """ + def _chunk_iter(): offset = len(text) while True: @@ -158,6 +159,7 @@ def memoizemethod_noargs(method): if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] + return new_method @@ -273,6 +275,19 @@ def equal_attributes(obj1, obj2, attributes): return True +@deprecated +class WeakKeyCache: + + def __init__(self, default_factory): + self.default_factory = default_factory + self._weakdict = weakref.WeakKeyDictionary() + + def __getitem__(self, key): + if key not in self._weakdict: + self._weakdict[key] = self.default_factory(key) + return self._weakdict[key] + + @deprecated def retry_on_eintr(function, *args, **kw): """Run a function and retry it while getting EINTR errors""" From a3fecaf07f9edd6a1bbac1ade825c23845c7e6b1 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 30 Jul 2020 15:45:27 +0530 Subject: [PATCH 211/568] test: fix host-name H2DownloadHandler tests --- tests/test_downloader_handlers.py | 33 +++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index f6add82dc..4dd32b6f5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -312,16 +312,18 @@ class HttpTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): - def _test(response): - self.assertEqual(response.body, b'example.com') - self.assertEqual(request.headers.get('Host'), b'example.com') + host = self.host + ':' + str(self.portno) - request = Request(self.getURL('host'), headers={'Host': 'example.com'}) + def _test(response): + self.assertEqual(response.body, bytes(host, 'utf-8')) + self.assertEqual(request.headers.get('Host'), bytes(host, 'utf-8')) + + request = Request(self.getURL('host'), headers={'Host': host}) return self.download_request(request, Spider('foo')).addCallback(_test) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b'example.com') + d.addCallback(self.assertEqual, b'localhost') return d def test_content_length_zero_bodyless_post_request_headers(self): @@ -339,7 +341,7 @@ class HttpTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.body, b'0') - request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'}) + request = Request(self.getURL('contentlength'), method='POST') return self.download_request(request, Spider('foo')).addCallback(_test) def test_content_length_zero_bodyless_post_only_one(self): @@ -525,6 +527,25 @@ class Https2TestCase(Https11TestCase): download_handler_cls = H2DownloadHandler HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + @defer.inlineCallbacks + def test_download_with_maxsize_very_large_file(self): + with mock.patch('scrapy.core.http2.stream.logger') as logger: + request = Request(self.getURL('largechunkedfile')) + + def check(logger): + logger.error.assert_called_once_with(mock.ANY) + + d = self.download_request(request, Spider('foo', download_maxsize=1500)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + # As the error message is logged in the dataReceived callback, we + # have to give a bit of time to the reactor to process the queue + # after closing the connection. + d = defer.Deferred() + d.addCallback(check) + reactor.callLater(.1, d.callback, logger) + yield d + def test_download_broken_content_cause_data_loss(self, url='broken'): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) From 890b2138a605af2bfbf340a0d48d9d83c4cda53b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Jul 2020 13:39:30 +0200 Subject: [PATCH 212/568] Remove the u prefix from strings --- docs/_ext/scrapydocs.py | 2 +- docs/topics/loaders.rst | 6 +- docs/topics/selectors.rst | 4 +- docs/utils/linkfix.py | 2 +- scrapy/http/request/form.py | 6 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/logformatter.py | 2 +- tests/test_cmdline/__init__.py | 2 +- tests/test_downloader_handlers.py | 6 +- tests/test_downloadermiddleware_cookies.py | 16 +- tests/test_downloadermiddleware_httpproxy.py | 6 +- tests/test_downloadermiddleware_redirect.py | 6 +- tests/test_downloadermiddleware_robotstxt.py | 4 +- tests/test_exporters.py | 84 +++---- tests/test_feedexport.py | 8 +- tests/test_http_headers.py | 6 +- tests/test_http_request.py | 90 ++++---- tests/test_http_response.py | 72 +++--- tests/test_item.py | 42 ++-- tests/test_linkextractors.py | 180 +++++++-------- tests/test_loader.py | 124 +++++----- tests/test_loader_deprecated.py | 226 +++++++++---------- tests/test_logformatter.py | 12 +- tests/test_mail.py | 8 +- tests/test_responsetypes.py | 10 +- tests/test_robotstxt_interface.py | 8 +- tests/test_selector.py | 32 +-- tests/test_spider.py | 12 +- tests/test_utils_iterators.py | 90 ++++---- tests/test_utils_python.py | 16 +- tests/test_utils_reqser.py | 2 +- tests/test_utils_template.py | 4 +- 32 files changed, 545 insertions(+), 545 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 192123473..640660943 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -17,7 +17,7 @@ class SettingsListDirective(Directive): def is_setting_index(node): if node.tagname == 'index': # index entries for setting directives look like: - # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')] + # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] entry_type, info, refid = node['entries'][0][:3] return entry_type == 'pair' and info.endswith('; setting') return False diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index d0eeb4097..29d9c5805 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -237,10 +237,10 @@ metadata. Here is an example:: >>> from scrapy.loader import ItemLoader >>> il = ItemLoader(item=Product()) ->>> il.add_value('name', [u'Welcome to my', u'website']) ->>> il.add_value('price', [u'€', u'1000']) +>>> il.add_value('name', ['Welcome to my', 'website']) +>>> il.add_value('price', ['€', '1000']) >>> il.load_item() -{'name': u'Welcome to my website', 'price': u'1000'} +{'name': 'Welcome to my website', 'price': '1000'} The precedence order, for both input and output processors, is as follows: diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index bb46ea80f..5014df6ac 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -734,7 +734,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's Example selecting links in list item with a "class" attribute ending with a digit: >>> from scrapy import Selector ->>> doc = u""" +>>> doc = """ ...
...
    ...
  • first item
    • @@ -765,7 +765,7 @@ extracting text elements for example. Example extracting microdata (sample content taken from https://schema.org/Product) with groups of itemscopes and corresponding itemprops:: - >>> doc = u""" + >>> doc = """ ...
    ... Kenmore White 17" Microwave ... Kenmore 17" Microwave diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 9acfc3b23..95a3f17d5 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -23,7 +23,7 @@ def main(): _contents = None # A regex that matches standard linkcheck output lines - line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') + line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') # Read lines from the linkcheck output file try: diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 0e6ceef0b..a260798ac 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -133,7 +133,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={ "re": "http://exslt.org/regular-expressions"}) - values = [(k, u'' if v is None else v) + values = [(k, '' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys] @@ -168,7 +168,7 @@ def _select_value(ele, n, v): # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') - v = [(o.get('value') or o.text or u'').strip() for o in selected_options] + v = [(o.get('value') or o.text or '').strip() for o in selected_options] return n, v @@ -205,7 +205,7 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items()) + xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 1615d44d7..8b9f961ee 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -76,7 +76,7 @@ class LxmlParserLinkExtractor: url = safe_url_string(url, encoding=response_encoding) # to fix relative links after process_value url = urljoin(response_url, url) - link = Link(url, _collect_string_content(el) or u'', + link = Link(url, _collect_string_content(el) or '', nofollow=rel_has_nofollow(el.get('rel'))) links.append(link) return self._deduplicate_if_needed(links) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 219145f13..0f9e6f1cb 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -44,7 +44,7 @@ class LogFormatter: def dropped(self, item, exception, response, spider): return { 'level': logging.INFO, # lowering the level from logging.WARNING - 'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s", + 'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s", 'args': { 'exception': exception, 'item': item, diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index da99a6be8..591075a98 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -59,7 +59,7 @@ class CmdlineTest(unittest.TestCase): 'EXTENSIONS=' + json.dumps(EXTENSIONS)) # XXX: There's gotta be a smarter way to do this... self.assertNotIn("...", settingsstr) - for char in ("'", "<", ">", 'u"'): + for char in ("'", "<", ">"): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 51deb20f4..57d4cdd6b 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1110,7 +1110,7 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype(self): def _test(response): - self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.text, '\u038e\u03a3\u038e') self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "iso-8859-7") @@ -1119,7 +1119,7 @@ class DataURITestCase(unittest.TestCase): def test_text_charset(self): def _test(response): - self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.text, '\u038e\u03a3\u038e') self.assertEqual(response.body, b'\xbe\xd3\xbe') self.assertEqual(response.encoding, "iso-8859-7") @@ -1128,7 +1128,7 @@ class DataURITestCase(unittest.TestCase): def test_mediatype_parameters(self): def _test(response): - self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.text, '\u038e\u03a3\u038e') self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "utf-8") diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 9ccc2110b..010577415 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -277,33 +277,33 @@ class CookiesMiddlewareTest(TestCase): def test_request_cookies_encoding(self): # 1) UTF8-encoded bytes - req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')}) + req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') # 2) Non UTF8-encoded bytes - req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')}) + req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')}) assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') - # 3) Unicode string - req3 = Request('http://example.org', cookies={'a': u'á'}) + # 3) String + req3 = Request('http://example.org', cookies={'a': 'á'}) assert self.mw.process_request(req3, self.spider) is None self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes - req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')}) + req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') # 2) Non UTF8-encoded bytes - req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')}) + req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')}) assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') - # 3) Unicode string - req3 = Request('http://example.org', headers={'Cookie': u'a=á'}) + # 3) String + req3 = Request('http://example.org', headers={'Cookie': 'a=á'}) assert self.mw.process_request(req3, self.spider) is None self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 9841d7a76..351631eb8 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -88,7 +88,7 @@ class TestHttpProxyMiddleware(TestCase): def test_proxy_auth_encoding(self): # utf-8 encoding - os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128' + os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128' mw = HttpProxyMiddleware(auth_encoding='utf-8') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None @@ -96,7 +96,7 @@ class TestHttpProxyMiddleware(TestCase): self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') # proxy from request.meta - req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') @@ -109,7 +109,7 @@ class TestHttpProxyMiddleware(TestCase): self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') # proxy from request.meta, latin-1 encoding - req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 919dbed23..131332131 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -184,7 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_latin1_location(self): req = Request('http://scrapytest.org/first') - latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1 + latin1_location = '/ação'.encode('latin1') # HTTP historically supports latin1 resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o' @@ -192,7 +192,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_utf8_location(self): req = Request('http://scrapytest.org/first') - utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encoding + utf8_location = '/ação'.encode('utf-8') # header using UTF-8 encoding resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' @@ -207,7 +207,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.mw = MetaRefreshMiddleware.from_crawler(crawler) def _body(self, interval=5, url='http://example.org/newpage'): - html = u"""""" + html = """""" return html.format(interval, url).encode('utf-8') def test_priority_adjust(self): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index b9452a0e7..f9936baba 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -30,7 +30,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def _get_successful_crawler(self): crawler = self.crawler crawler.settings.set('ROBOTSTXT_OBEY', True) - ROBOTS = u""" + ROBOTS = """ User-Agent: * Disallow: /admin/ Disallow: /static/ @@ -56,7 +56,7 @@ Disallow: /some/randome/page.html self.assertIgnored(Request('http://site.local/admin/main'), middleware), self.assertIgnored(Request('http://site.local/static/'), middleware), self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware), - self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware) + self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware) ], fireOnOneErrback=True) def test_robotstxt_ready_parser(self): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 25da54a65..660c99ce1 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -54,7 +54,7 @@ class BaseItemExporterTest(unittest.TestCase): def setUp(self): if self.item_class is None: raise unittest.SkipTest("item class is None") - self.i = self.item_class(name=u'John\xa3', age=u'22') + self.i = self.item_class(name='John\xa3', age='22') self.output = BytesIO() self.ie = self._get_exporter() @@ -96,25 +96,25 @@ class BaseItemExporterTest(unittest.TestCase): def test_serialize_field(self): a = ItemAdapter(self.i) res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name']) - self.assertEqual(res, u'John\xa3') + self.assertEqual(res, 'John\xa3') res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age']) - self.assertEqual(res, u'22') + self.assertEqual(res, '22') def test_fields_to_export(self): ie = self._get_exporter(fields_to_export=['name']) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')]) + self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')]) ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') _, name = list(ie._get_serialized_fields(self.i))[0] assert isinstance(name, str) - self.assertEqual(name, u'John\xa3') + self.assertEqual(name, 'John\xa3') def test_field_custom_serializer(self): - i = self.custom_field_item_class(name=u'John\xa3', age=u'22') + i = self.custom_field_item_class(name='John\xa3', age='22') a = ItemAdapter(i) ie = self._get_exporter() - self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3') + self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3') self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24') @@ -132,48 +132,48 @@ class PythonItemExporterTest(BaseItemExporterTest): PythonItemExporter(invalid_option='something') def test_nested_item(self): - i1 = self.item_class(name=u'Joseph', age='22') - i2 = dict(name=u'Maria', age=i1) - i3 = self.item_class(name=u'Jesus', age=i2) + i1 = self.item_class(name='Joseph', age='22') + i2 = dict(name='Maria', age=i1) + i3 = self.item_class(name='Jesus', age=i2) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual(type(exported), dict) self.assertEqual( exported, - {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'} + {'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'} ) self.assertEqual(type(exported['age']), dict) self.assertEqual(type(exported['age']['age']), dict) def test_export_list(self): - i1 = self.item_class(name=u'Joseph', age='22') - i2 = self.item_class(name=u'Maria', age=[i1]) - i3 = self.item_class(name=u'Jesus', age=[i2]) + i1 = self.item_class(name='Joseph', age='22') + i2 = self.item_class(name='Maria', age=[i1]) + i3 = self.item_class(name='Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( exported, - {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'} ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_item_dict_list(self): - i1 = self.item_class(name=u'Joseph', age='22') - i2 = dict(name=u'Maria', age=[i1]) - i3 = self.item_class(name=u'Jesus', age=[i2]) + i1 = self.item_class(name='Joseph', age='22') + i2 = dict(name='Maria', age=[i1]) + i3 = self.item_class(name='Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( exported, - {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'} ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_binary(self): exporter = PythonItemExporter(binary=True) - value = self.item_class(name=u'John\xa3', age=u'22') + value = self.item_class(name='John\xa3', age='22') expected = {b'name': b'John\xc2\xa3', b'age': b'22'} self.assertEqual(expected, exporter.export_item(value)) @@ -279,7 +279,7 @@ class CsvItemExporterTest(BaseItemExporterTest): return self.assertEqual(split_csv(first), split_csv(second), msg=msg) def _check_output(self): - self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n') + self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n') def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() @@ -396,7 +396,7 @@ class XmlItemExporterTest(BaseItemExporterTest): def test_multivalued_fields(self): self.assertExportResult( - self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]), + self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]), b"""\n @@ -408,9 +408,9 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_item(self): - i1 = dict(name=u'foo\xa3hoo', age='22') - i2 = dict(name=u'bar', age=i1) - i3 = self.item_class(name=u'buz', age=i2) + i1 = dict(name='foo\xa3hoo', age='22') + i2 = dict(name='bar', age=i1) + i3 = self.item_class(name='buz', age=i2) self.assertExportResult( i3, @@ -431,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_list_item(self): - i1 = dict(name=u'foo') - i2 = dict(name=u'bar', v2={"egg": ["spam"]}) - i3 = self.item_class(name=u'buz', age=[i1, i2]) + i1 = dict(name='foo') + i2 = dict(name='bar', v2={"egg": ["spam"]}) + i3 = self.item_class(name='buz', age=[i1, i2]) self.assertExportResult( i3, @@ -475,7 +475,7 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest): class JsonLinesItemExporterTest(BaseItemExporterTest): - _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}} + _expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}} def _get_exporter(self, **kwargs): return JsonLinesItemExporter(self.output, **kwargs) @@ -485,9 +485,9 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.assertEqual(exported, ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = self.item_class(name=u'Joseph', age='22') - i2 = dict(name=u'Maria', age=i1) - i3 = self.item_class(name=u'Jesus', age=i2) + i1 = self.item_class(name='Joseph', age='22') + i2 = dict(name='Maria', age=i1) + i3 = self.item_class(name='Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() @@ -542,25 +542,25 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = self.item_class(name=u'Joseph\xa3', age='22') - i2 = self.item_class(name=u'Maria', age=i1) - i3 = self.item_class(name=u'Jesus', age=i2) + i1 = self.item_class(name='Joseph\xa3', age='22') + i2 = self.item_class(name='Maria', age=i1) + i3 = self.item_class(name='Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} + expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} self.assertEqual(exported, [expected]) def test_nested_dict_item(self): - i1 = dict(name=u'Joseph\xa3', age='22') - i2 = self.item_class(name=u'Maria', age=i1) - i3 = dict(name=u'Jesus', age=i2) + i1 = dict(name='Joseph\xa3', age='22') + i2 = self.item_class(name='Maria', age=i1) + i3 = dict(name='Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}} + expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}} self.assertEqual(exported, [expected]) def test_nonstring_types_item(self): @@ -595,14 +595,14 @@ class CustomExporterItemTest(unittest.TestCase): else: return super(CustomItemExporter, self).serialize_field(field, name, value) - i = self.item_class(name=u'John', age='22') + i = self.item_class(name='John', age='22') a = ItemAdapter(i) ie = CustomItemExporter() self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John') self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23') - i2 = {'name': u'John', 'age': '22'} + i2 = {'name': 'John', 'age': '22'} self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John') self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23') diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c49b2e92f..b57349848 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -874,7 +874,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_encoding(self): - items = [dict({'foo': u'Test\xd6'})] + items = [dict({'foo': 'Test\xd6'})] formats = { 'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'), @@ -919,7 +919,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): - items = [dict({'foo': u'FOO', 'bar': u'BAR'})] + items = [dict({'foo': 'FOO', 'bar': 'BAR'})] formats = { 'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'), @@ -1393,7 +1393,7 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): - items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] + items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})] formats = { 'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'), @@ -1442,7 +1442,7 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): - items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})] + items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})] formats = { 'json': ['[{"foo": "FOO"}]'.encode('utf-8'), '[{"foo": "FOO1"}]'.encode('utf-8')], diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index cf3fc8496..64ff7a73d 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -39,19 +39,19 @@ class HeadersTest(unittest.TestCase): assert h.getlist('X-Forwarded-For') is not hlist def test_encode_utf8(self): - h = Headers({u'key': u'\xa3'}, encoding='utf-8') + h = Headers({'key': '\xa3'}, encoding='utf-8') key, val = dict(h).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] self.assertEqual(val[0], b'\xc2\xa3') def test_encode_latin1(self): - h = Headers({u'key': u'\xa3'}, encoding='latin1') + h = Headers({'key': '\xa3'}, encoding='latin1') key, val = dict(h).popitem() self.assertEqual(val[0], b'\xa3') def test_encode_multiple(self): - h = Headers({u'key': [u'\xa3']}, encoding='utf-8') + h = Headers({'key': ['\xa3']}, encoding='utf-8') key, val = dict(h).popitem() self.assertEqual(val[0], b'\xc2\xa3') diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 63014b22d..f5cf4e798 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -60,8 +60,8 @@ class RequestTest(unittest.TestCase): self.assertFalse(p.headers is r.headers) # headers must not be unicode - h = Headers({'key1': u'val1', u'key2': 'val2'}) - h[u'newkey'] = u'newval' + h = Headers({'key1': 'val1', 'key2': 'val2'}) + h['newkey'] = 'newval' for k, v in h.items(): self.assertIsInstance(k, bytes) for s in v: @@ -89,30 +89,30 @@ class RequestTest(unittest.TestCase): self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") def test_url_encoding(self): - r = self.request_class(url=u"http://www.scrapy.org/price/£") + r = self.request_class(url="http://www.scrapy.org/price/£") self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") def test_url_encoding_other(self): # encoding affects only query part of URI, not path # path part should always be UTF-8 encoded before percent-escaping - r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8") + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") - r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1") + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") def test_url_encoding_query(self): - r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ") + r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") # should be same as above - r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8") + r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8") self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") def test_url_encoding_query_latin1(self): # encoding is used for encoding query-string before percent-escaping; # path is still UTF-8 encoded before percent-escaping - r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1") + r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1") self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3") def test_url_encoding_nonutf8_untouched(self): @@ -131,16 +131,16 @@ class RequestTest(unittest.TestCase): # characters. Otherwise, in the future the IRI will be mapped to # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different # URI from "http://www.example.org/r%E9sum%E9.html". - r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3") + r1 = self.request_class(url="http://www.scrapy.org/price/%a3") self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3") - r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3") + r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html") + r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html") def test_body(self): @@ -151,11 +151,11 @@ class RequestTest(unittest.TestCase): assert isinstance(r2.body, bytes) self.assertEqual(r2.encoding, 'utf-8') # default encoding - r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8') + r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8') assert isinstance(r3.body, bytes) self.assertEqual(r3.body, b"Price: \xc2\xa3100") - r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1') + r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1') assert isinstance(r4.body, bytes) self.assertEqual(r4.body, b"Price: \xa3100") @@ -164,7 +164,7 @@ class RequestTest(unittest.TestCase): r = self.request_class(url="http://www.example.com/ajax.html#!key=value") self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue") # unicode url - r = self.request_class(url=u"http://www.example.com/ajax.html#!key=value") + r = self.request_class(url="http://www.example.com/ajax.html#!key=value") self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue") def test_copy(self): @@ -236,7 +236,7 @@ class RequestTest(unittest.TestCase): assert r4.dont_filter is False def test_method_always_str(self): - r = self.request_class("http://www.example.com", method=u"POST") + r = self.request_class("http://www.example.com", method="POST") assert isinstance(r.method, str) def test_immutable_attributes(self): @@ -381,7 +381,7 @@ class FormRequestTest(RequestTest): def test_default_encoding_textual_data(self): # using default encoding (utf-8) - data = {u'µ one': u'two', u'price': u'£ 100'} + data = {'µ one': 'two', 'price': '£ 100'} r2 = self.request_class("http://www.example.com", formdata=data) self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'utf-8') @@ -390,7 +390,7 @@ class FormRequestTest(RequestTest): def test_default_encoding_mixed_data(self): # using default encoding (utf-8) - data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'} + data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'} r2 = self.request_class("http://www.example.com", formdata=data) self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'utf-8') @@ -406,14 +406,14 @@ class FormRequestTest(RequestTest): self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') def test_custom_encoding_textual_data(self): - data = {'price': u'£ 100'} + data = {'price': '£ 100'} r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') self.assertEqual(r3.encoding, 'latin1') self.assertEqual(r3.body, b'price=%A3+100') def test_multi_key_values(self): # using multiples values for a single key - data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']} + data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']} r3 = self.request_class("http://www.example.com", formdata=data) self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100') @@ -450,10 +450,10 @@ class FormRequestTest(RequestTest): self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) - self.assertEqual(set(fs[u'one']), {u'two', u'three'}) - self.assertEqual(fs[u'test2'], [u'xxx µ']) - self.assertEqual(fs[u'six'], [u'seven']) + self.assertEqual(set(fs['test £']), {'val1', 'val2'}) + self.assertEqual(set(fs['one']), {'two', 'three'}) + self.assertEqual(fs['test2'], ['xxx µ']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_post_nonascii_bytes_latin1(self): response = _buildresponse( @@ -471,14 +471,14 @@ class FormRequestTest(RequestTest): self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True, encoding='latin1') - self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) - self.assertEqual(set(fs[u'one']), {u'two', u'three'}) - self.assertEqual(fs[u'test2'], [u'xxx µ']) - self.assertEqual(fs[u'six'], [u'seven']) + self.assertEqual(set(fs['test £']), {'val1', 'val2'}) + self.assertEqual(set(fs['one']), {'two', 'three'}) + self.assertEqual(fs['test2'], ['xxx µ']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_post_nonascii_unicode(self): response = _buildresponse( - u""" + """ @@ -490,10 +490,10 @@ class FormRequestTest(RequestTest): self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) - self.assertEqual(set(fs[u'one']), {u'two', u'three'}) - self.assertEqual(fs[u'test2'], [u'xxx µ']) - self.assertEqual(fs[u'six'], [u'seven']) + self.assertEqual(set(fs['test £']), {'val1', 'val2'}) + self.assertEqual(set(fs['one']), {'two', 'three'}) + self.assertEqual(fs['test2'], ['xxx µ']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_duplicate_form_key(self): response = _buildresponse( @@ -685,7 +685,7 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response( - response, clickdata={u'name': u'clickable', u'value': u'clicked2'} + response, clickdata={'name': 'clickable', 'value': 'clicked2'} ) fs = _qs(req) self.assertEqual(fs[b'clickable'], [b'clicked2']) @@ -694,21 +694,21 @@ class FormRequestTest(RequestTest): def test_from_response_unicode_clickdata(self): response = _buildresponse( - u"""
    + """
    """) req = self.request_class.from_response( - response, clickdata={u'name': u'price in \u00a3'} + response, clickdata={'name': 'price in \u00a3'} ) fs = _qs(req, to_unicode=True) - self.assertTrue(fs[u'price in \u00a3']) + self.assertTrue(fs['price in \u00a3']) def test_from_response_unicode_clickdata_latin1(self): response = _buildresponse( - u"""
    + """ @@ -716,10 +716,10 @@ class FormRequestTest(RequestTest):
    """, encoding='latin1') req = self.request_class.from_response( - response, clickdata={u'name': u'price in \u00a5'} + response, clickdata={'name': 'price in \u00a5'} ) fs = _qs(req, to_unicode=True, encoding='latin1') - self.assertTrue(fs[u'price in \u00a5']) + self.assertTrue(fs['price in \u00a5']) def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( @@ -733,7 +733,7 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response( - response, formname='form2', clickdata={u'name': u'clickable'} + response, formname='form2', clickdata={'name': 'clickable'} ) fs = _qs(req) self.assertEqual(fs[b'clickable'], [b'clicked2']) @@ -1072,11 +1072,11 @@ class FormRequestTest(RequestTest): def test_from_response_unicode_xpath(self): response = _buildresponse(b'
    ') - r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']") + r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']") fs = _qs(r) self.assertEqual(fs, {}) - xpath = u"//form[@name='\u03b1']" + xpath = "//form[@name='\u03b1']" self.assertRaisesRegex(ValueError, re.escape(xpath), self.request_class.from_response, response, formxpath=xpath) @@ -1246,13 +1246,13 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=('value',)) self._test_request(params=('username', 'password'), methodname='login') self._test_request(params=('response', ), methodresponse='login') - self._test_request(params=(u'pas£',), encoding='utf-8') + self._test_request(params=('pas£',), encoding='utf-8') self._test_request(params=(None,), allow_none=1) self.assertRaises(TypeError, self._test_request) self.assertRaises(TypeError, self._test_request, params=(None,)) def test_latin1(self): - self._test_request(params=(u'pas£',), encoding='latin1') + self._test_request(params=('pas£',), encoding='latin1') class JsonRequestTest(RequestTest): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index e0ca3c0e6..56d017de6 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -318,28 +318,28 @@ class TextResponseTest(BaseResponseTest): def test_unicode_url(self): # instantiate with unicode url without encoding (should set default encoding) - resp = self.response_class(u"http://www.example.com/") + resp = self.response_class("http://www.example.com/") self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) # make sure urls are converted to str - resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8') + resp = self.response_class(url="http://www.example.com/", encoding='utf-8') assert isinstance(resp.url, str) - resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8') + resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8') self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1') + resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1') self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') - resp = self.response_class(u"http://www.example.com/price/\xa3", + resp = self.response_class("http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(u"http://www.example.com/price/\xa3", + resp = self.response_class("http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') def test_unicode_body(self): unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 ' '\u0442\u0435\u043a\u0441\u0442') - self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body') + self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body') original_string = unicode_string.encode('cp1251') r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') @@ -355,7 +355,7 @@ class TextResponseTest(BaseResponseTest): def test_encoding(self): r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) - r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") + r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3") r3 = self.response_class("http://www.example.com", body=b"\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") @@ -376,14 +376,14 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r5._headers_encoding(), None) self._assert_response_encoding(r5, "utf-8") assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' - self._assert_response_values(r1, 'utf-8', u"\xa3") - self._assert_response_values(r2, 'utf-8', u"\xa3") - self._assert_response_values(r3, 'iso-8859-1', u"\xa3") - self._assert_response_values(r6, 'gb18030', u"\u2015") - self._assert_response_values(r7, 'gb18030', u"\u2015") + self._assert_response_values(r1, 'utf-8', "\xa3") + self._assert_response_values(r2, 'utf-8', "\xa3") + self._assert_response_values(r3, 'iso-8859-1', "\xa3") + self._assert_response_values(r6, 'gb18030', "\u2015") + self._assert_response_values(r7, 'gb18030', "\u2015") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies - self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3") + self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3") def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" @@ -391,14 +391,14 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=UKNOWN"]}, body=b"\xc2\xa3") self.assertEqual(r._declared_encoding(), None) - self._assert_response_values(r, 'utf-8', u"\xa3") + self._assert_response_values(r, 'utf-8', "\xa3") def test_utf16(self): """Test utf-16 because UnicodeDammit is known to have problems with""" r = self.response_class("http://www.example.com", body=b'\xff\xfeh\x00i\x00', encoding='utf-16') - self._assert_response_values(r, 'utf-16', u"hi") + self._assert_response_values(r, 'utf-16', "hi") def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): r6 = self.response_class("http://www.example.com", @@ -406,8 +406,8 @@ class TextResponseTest(BaseResponseTest): body=b"\xef\xbb\xbfWORD\xe3\xab") self.assertEqual(r6.encoding, 'utf-8') self.assertIn(r6.text, { - u'WORD\ufffd\ufffd', # w3lib < 1.19.0 - u'WORD\ufffd', # w3lib >= 1.19.0 + 'WORD\ufffd\ufffd', # w3lib < 1.19.0 + 'WORD\ufffd', # w3lib >= 1.19.0 }) def test_bom_is_removed_from_body(self): @@ -422,9 +422,9 @@ class TextResponseTest(BaseResponseTest): # Test response without content-type and BOM encoding response = self.response_class(url, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') response = self.response_class(url, body=body) - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') self.assertEqual(response.encoding, 'utf-8') # Body caching sideeffect isn't triggered when encoding is declared in @@ -432,28 +432,28 @@ class TextResponseTest(BaseResponseTest): # body response = self.response_class(url, headers=headers, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') self.assertEqual(response.encoding, 'utf-8') def test_replace_wrong_encoding(self): """Test invalid chars are replaced properly""" r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX') # XXX: Policy for replacing invalid chars may suffer minor variations - # but it should always contain the unicode replacement char (u'\ufffd') - assert u'\ufffd' in r.text, repr(r.text) - assert u'PREFIX' in r.text, repr(r.text) - assert u'SUFFIX' in r.text, repr(r.text) + # but it should always contain the unicode replacement char ('\ufffd') + assert '\ufffd' in r.text, repr(r.text) + assert 'PREFIX' in r.text, repr(r.text) + assert 'SUFFIX' in r.text, repr(r.text) # Do not destroy html tags due to encoding bugs r = self.response_class("http://example.com", encoding='utf-8', body=b'\xf0value') - assert u'value' in r.text, repr(r.text) + assert 'value' in r.text, repr(r.text) # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') - # assert u'\ufffd' in r.text, repr(r.text) + # assert '\ufffd' in r.text, repr(r.text) def test_selector(self): body = b"Some page" @@ -466,15 +466,15 @@ class TextResponseTest(BaseResponseTest): self.assertEqual( response.selector.xpath("//title/text()").getall(), - [u'Some page'] + ['Some page'] ) self.assertEqual( response.selector.css("title::text").getall(), - [u'Some page'] + ['Some page'] ) self.assertEqual( response.selector.re("Some (.*)"), - [u'page'] + ['page'] ) def test_selector_shortcuts(self): @@ -595,7 +595,7 @@ class TextResponseTest(BaseResponseTest): resp1 = self.response_class( 'http://example.com', encoding='utf8', - body=u'click me'.encode('utf8') + body='click me'.encode('utf8') ) req = self._assert_followed_url( resp1.css('a')[0], @@ -607,7 +607,7 @@ class TextResponseTest(BaseResponseTest): resp2 = self.response_class( 'http://example.com', encoding='cp1251', - body=u'click me'.encode('cp1251') + body='click me'.encode('cp1251') ) req = self._assert_followed_url( resp2.css('a')[0], @@ -681,8 +681,8 @@ class TextResponseTest(BaseResponseTest): def test_body_as_unicode_deprecation_warning(self): with catch_warnings(record=True) as warnings: - r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8') - self.assertEqual(r1.body_as_unicode(), u'Hello') + r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8') + self.assertEqual(r1.body_as_unicode(), 'Hello') self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) @@ -787,7 +787,7 @@ class XmlResponseTest(TextResponseTest): self.assertEqual( response.selector.xpath("//elem/text()").getall(), - [u'value'] + ['value'] ) def test_selector_shortcuts(self): diff --git a/tests/test_item.py b/tests/test_item.py index 60468971c..0ce78f8c0 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -20,8 +20,8 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - i['name'] = u'name' - self.assertEqual(i['name'], u'name') + i['name'] = 'name' + self.assertEqual(i['name'], 'name') def test_init(self): class TestItem(Item): @@ -30,17 +30,17 @@ class ItemTest(unittest.TestCase): i = TestItem() self.assertRaises(KeyError, i.__getitem__, 'name') - i2 = TestItem(name=u'john doe') - self.assertEqual(i2['name'], u'john doe') + i2 = TestItem(name='john doe') + self.assertEqual(i2['name'], 'john doe') - i3 = TestItem({'name': u'john doe'}) - self.assertEqual(i3['name'], u'john doe') + i3 = TestItem({'name': 'john doe'}) + self.assertEqual(i3['name'], 'john doe') i4 = TestItem(i3) - self.assertEqual(i4['name'], u'john doe') + self.assertEqual(i4['name'], 'john doe') - self.assertRaises(KeyError, TestItem, {'name': u'john doe', - 'other': u'foo'}) + self.assertRaises(KeyError, TestItem, {'name': 'john doe', + 'other': 'foo'}) def test_invalid_field(self): class TestItem(Item): @@ -56,7 +56,7 @@ class ItemTest(unittest.TestCase): number = Field() i = TestItem() - i['name'] = u'John Doe' + i['name'] = 'John Doe' i['number'] = 123 itemrepr = repr(i) @@ -101,9 +101,9 @@ class ItemTest(unittest.TestCase): i = TestItem() self.assertRaises(KeyError, i.get_name) - i['name'] = u'lala' - self.assertEqual(i.get_name(), u'lala') - i.change_name(u'other') + i['name'] = 'lala' + self.assertEqual(i.get_name(), 'lala') + i.change_name('other') self.assertEqual(i.get_name(), 'other') def test_metaclass(self): @@ -113,22 +113,22 @@ class ItemTest(unittest.TestCase): values = Field() i = TestItem() - i['name'] = u'John' + i['name'] = 'John' self.assertEqual(list(i.keys()), ['name']) self.assertEqual(list(i.values()), ['John']) - i['keys'] = u'Keys' - i['values'] = u'Values' + i['keys'] = 'Keys' + i['values'] = 'Values' self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name']) - self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John']) + self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John']) def test_metaclass_with_fields_attribute(self): class TestItem(Item): fields = {'new': Field(default='X')} - item = TestItem(new=u'New') + item = TestItem(new='New') self.assertSortedEqual(list(item.keys()), ['new']) - self.assertSortedEqual(list(item.values()), [u'New']) + self.assertSortedEqual(list(item.values()), ['New']) def test_metaclass_inheritance(self): class ParentItem(Item): @@ -238,8 +238,8 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - i['name'] = u'John' - self.assertEqual(dict(i), {'name': u'John'}) + i['name'] = 'John' + self.assertEqual(dict(i), {'name': 'John'}) def test_copy(self): class TestItem(Item): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 8d4538eed..a0bafa5e5 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -31,31 +31,31 @@ class Base: page4_url = 'http://example.com/page%204.html' self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url=page4_url, text=u'href with whitespaces'), + Link(url='http://www.google.com/something', text=''), + Link(url='http://example.com/innertag.html', text='inner tag'), + Link(url=page4_url, text='href with whitespaces'), ]) def test_extract_filter_allow(self): lx = self.extractor_cls(allow=('sample', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) def test_extract_filter_allow_with_duplicates(self): lx = self.extractor_cls(allow=('sample', ), unique=False) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) @@ -63,10 +63,10 @@ class Base: lx = self.extractor_cls(allow=('sample', ), unique=False, canonicalize=True) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment') ]) @@ -74,22 +74,22 @@ class Base: lx = self.extractor_cls(allow=('sample',), unique=True, canonicalize=True) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), ]) def test_extract_filter_allow_and_deny(self): lx = self.extractor_cls(allow=('sample', ), deny=('3', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) def test_extract_filter_allowed_domains(self): lx = self.extractor_cls(allow_domains=('google.com', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), + Link(url='http://www.google.com/something', text=''), ]) def test_extraction_using_single_values(self): @@ -97,27 +97,27 @@ class Base: lx = self.extractor_cls(allow='sample') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) lx = self.extractor_cls(allow='sample', deny='3') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) lx = self.extractor_cls(allow_domains='google.com') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), + Link(url='http://www.google.com/something', text=''), ]) lx = self.extractor_cls(deny_domains='example.com') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), + Link(url='http://www.google.com/something', text=''), ]) def test_nofollow(self): @@ -145,11 +145,11 @@ class Base: lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/about.html', text=u'About us'), - Link(url='http://example.org/follow.html', text=u'Follow this link'), - Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True), - Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + Link(url='http://example.org/about.html', text='About us'), + Link(url='http://example.org/follow.html', text='Follow this link'), + Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True), + Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'), + Link(url='http://google.com/something', text='External link not to follow', nofollow=True), ]) def test_matches(self): @@ -183,8 +183,8 @@ class Base: def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) def test_restrict_xpaths_encoding(self): @@ -202,14 +202,14 @@ class Base: lx = self.extractor_cls(restrict_xpaths="//div[@class='links']") self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/about.html', text=u'About us\xa3')]) + [Link(url='http://example.org/about.html', text='About us\xa3')]) def test_restrict_xpaths_with_html_entities(self): html = b'

    text

    ' response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15') links = self.extractor_cls(restrict_xpaths='//p').extract_links(response) self.assertEqual(links, - [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')]) + [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')]) def test_restrict_xpaths_concat_in_handle_data(self): """html entities cause SGMLParser to call handle_data hook twice""" @@ -217,22 +217,22 @@ class Base: response = HtmlResponse("http://example.org", body=body, encoding='gb18030') lx = self.extractor_cls(restrict_xpaths="//div") self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c', + [Link(url='http://example.org/foo', text='>\u4eac<\u4e1c', fragment='', nofollow=False)]) def test_restrict_css(self): lx = self.extractor_cls(restrict_css=('#subwrapper a',)) self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2') + Link(url='http://example.com/sample2.html', text='sample 2') ]) def test_restrict_css_and_restrict_xpaths_together(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ), restrict_css=('#subwrapper + a', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), ]) def test_area_tag_with_unicode_present(self): @@ -243,7 +243,7 @@ class Base: lx.extract_links(response) lx.extract_links(response) self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/foo', text=u'', + [Link(url='http://example.org/foo', text='', fragment='', nofollow=False)]) def test_encoded_url(self): @@ -251,7 +251,7 @@ class Base: response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), + Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False), ]) def test_encoded_url_in_restricted_xpath(self): @@ -259,7 +259,7 @@ class Base: response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') lx = self.extractor_cls(restrict_xpaths="//div") self.assertEqual(lx.extract_links(response), [ - Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), + Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False), ]) def test_ignored_extensions(self): @@ -268,7 +268,7 @@ class Base: response = HtmlResponse("http://example.org/", body=html) lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/page.html', text=u'asd'), + Link(url='http://example.org/page.html', text='asd'), ]) # override denied extensions @@ -308,25 +308,25 @@ class Base: page4_url = 'http://example.com/page%204.html' self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url=page4_url, text=u'href with whitespaces'), + Link(url='http://www.google.com/something', text=''), + Link(url='http://example.com/innertag.html', text='inner tag'), + Link(url=page4_url, text='href with whitespaces'), ]) lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=()) self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample2.jpg', text=u''), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample2.jpg', text=''), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url=page4_url, text=u'href with whitespaces'), + Link(url='http://www.google.com/something', text=''), + Link(url='http://example.com/innertag.html', text='inner tag'), + Link(url=page4_url, text='href with whitespaces'), ]) lx = self.extractor_cls(attrs=None) @@ -344,24 +344,24 @@ class Base: lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) lx = self.extractor_cls(tags="area") self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample1.html', text=''), ]) lx = self.extractor_cls(tags="a") self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=()) self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample2.jpg', text=u''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample2.jpg', text=''), ]) def test_tags_attrs(self): @@ -375,14 +375,14 @@ class Base: lx = self.extractor_cls(tags='div', attrs='data-url') self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), - Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) + Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False), + Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False) ]) lx = self.extractor_cls(tags=('div',), attrs=('data-url',)) self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), - Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) + Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False), + Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False) ]) def test_xhtml(self): @@ -420,13 +420,13 @@ class Base: self.assertEqual( lx.extract_links(response), [ - Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', + Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text='Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', + Link(url='http://example.com/nofollow2.html', text='Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + Link(url='http://google.com/something', text='External link not to follow', nofollow=True), ] ) @@ -436,13 +436,13 @@ class Base: self.assertEqual( lx.extract_links(response), [ - Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', + Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text='Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', + Link(url='http://example.com/nofollow2.html', text='Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + Link(url='http://google.com/something', text='External link not to follow', nofollow=True), ] ) @@ -455,8 +455,8 @@ class Base: response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), + Link(url='http://example.org/item1.html', text='Item 1', nofollow=False), + Link(url='http://example.org/item3.html', text='Item 3', nofollow=False), ]) def test_ftp_links(self): @@ -467,7 +467,7 @@ class Base: response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8') lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False), + Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False), ]) def test_pickle_extractor(self): @@ -487,8 +487,8 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), + Link(url='http://example.org/item1.html', text='Item 1', nofollow=False), + Link(url='http://example.org/item3.html', text='Item 3', nofollow=False), ]) def test_link_restrict_text(self): @@ -501,18 +501,18 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Simple text inclusion test lx = self.extractor_cls(restrict_text='dog') self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), ]) # Unique regex test lx = self.extractor_cls(restrict_text=r'of.*dog') self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), ]) # Multiple regex test lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat']) self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False), - Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False), + Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), ]) def test_restrict_xpaths_with_html_entities(self): diff --git a/tests/test_loader.py b/tests/test_loader.py index 581183625..2ed6f365f 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -69,23 +69,23 @@ class BasicItemLoaderTest(unittest.TestCase): def test_add_value_on_unknown_field(self): il = TestItemLoader() - self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo']) + self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo']) def test_load_item_using_default_loader(self): i = TestItem() - i['summary'] = u'lala' + i['summary'] = 'lala' il = ItemLoader(item=i) - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() assert item is i - self.assertEqual(item['summary'], [u'lala']) - self.assertEqual(item['name'], [u'marta']) + self.assertEqual(item['summary'], ['lala']) + self.assertEqual(item['name'], ['marta']) def test_load_item_using_custom_loader(self): il = TestItemLoader() - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() - self.assertEqual(item['name'], [u'Marta']) + self.assertEqual(item['name'], ['Marta']) class InitializationTestMixin: @@ -290,137 +290,137 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertRaises(RuntimeError, l.get_css, '#name::text') def test_init_method_with_selector(self): - sel = Selector(text=u"
    marta
    ") + sel = Selector(text="
    marta
    ") l = TestItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) def test_init_method_with_selector_css(self): - sel = Selector(text=u"
    marta
    ") + sel = Selector(text="
    marta
    ") l = TestItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) def test_init_method_with_response(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) def test_init_method_with_response_css(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.add_css('url', 'a::attr(href)') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) # combining/accumulating CSS selectors and XPath expressions l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta', u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta']) l.add_xpath('url', '//img/@src') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org', u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png']) def test_add_xpath_re(self): l = TestItemLoader(response=self.response) l.add_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_output_value('name'), [u'Ma']) + self.assertEqual(l.get_output_value('name'), ['Ma']) def test_replace_xpath(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_xpath('name', '//p/text()') - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) l.replace_xpath('name', ['//p/text()', '//div/text()']) - self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta']) + self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta']) def test_get_xpath(self): l = TestItemLoader(response=self.response) - self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph']) - self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph') - self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa') + self.assertEqual(l.get_xpath('//p/text()'), ['paragraph']) + self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph') + self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa') - self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta']) + self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta']) def test_replace_xpath_multi_fields(self): l = TestItemLoader(response=self.response) l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) def test_replace_xpath_re(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_output_value('name'), [u'Ma']) + self.assertEqual(l.get_output_value('name'), ['Ma']) def test_add_css_re(self): l = TestItemLoader(response=self.response) l.add_css('name', 'div::text', re='ma') - self.assertEqual(l.get_output_value('name'), [u'Ma']) + self.assertEqual(l.get_output_value('name'), ['Ma']) l.add_css('url', 'a::attr(href)', re='http://(.+)') - self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['www.scrapy.org']) def test_replace_css(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_css('name', 'p::text') - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) l.replace_css('name', ['p::text', 'div::text']) - self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta']) + self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta']) l.add_css('url', 'a::attr(href)', re='http://(.+)') - self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['www.scrapy.org']) l.replace_css('url', 'img::attr(src)') - self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) def test_get_css(self): l = TestItemLoader(response=self.response) - self.assertEqual(l.get_css('p::text'), [u'paragraph']) - self.assertEqual(l.get_css('p::text', TakeFirst()), u'paragraph') - self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), u'pa') + self.assertEqual(l.get_css('p::text'), ['paragraph']) + self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph') + self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa') - self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta']) + self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta']) self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']), - [u'http://www.scrapy.org', u'/images/logo.png']) + ['http://www.scrapy.org', '/images/logo.png']) def test_replace_css_multi_fields(self): l = TestItemLoader(response=self.response) l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x}) - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x}) - self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) def test_replace_css_re(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css('url', 'a::attr(href)') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)') - self.assertEqual(l.get_output_value('url'), [u'scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['scrapy.org']) class SubselectorLoaderTest(unittest.TestCase): @@ -447,9 +447,9 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css('name_div', '#id') nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value('name'), [u'marta']) - self.assertEqual(l.get_output_value('name_div'), [u'
    marta
    ']) - self.assertEqual(l.get_output_value('name_value'), [u'marta']) + self.assertEqual(l.get_output_value('name'), ['marta']) + self.assertEqual(l.get_output_value('name_div'), ['
    marta
    ']) + self.assertEqual(l.get_output_value('name_value'), ['marta']) self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) @@ -462,9 +462,9 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css('name_div', '#id') nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value('name'), [u'marta']) - self.assertEqual(l.get_output_value('name_div'), [u'
    marta
    ']) - self.assertEqual(l.get_output_value('name_value'), [u'marta']) + self.assertEqual(l.get_output_value('name'), ['marta']) + self.assertEqual(l.get_output_value('name_div'), ['
    marta
    ']) + self.assertEqual(l.get_output_value('name_value'), ['marta']) self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) @@ -476,11 +476,11 @@ class SubselectorLoaderTest(unittest.TestCase): nl2 = nl1.nested_xpath('a') l.add_xpath('url', '//footer/a/@href') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) nl1.replace_xpath('url', 'img/@src') - self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) nl2.replace_xpath('url', '@href') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) def test_nested_ordering(self): l = NestedItemLoader(response=self.response) @@ -493,10 +493,10 @@ class SubselectorLoaderTest(unittest.TestCase): l.add_xpath('url', '//footer/a/@href') self.assertEqual(l.get_output_value('url'), [ - u'/images/logo.png', - u'http://www.scrapy.org', - u'homepage', - u'http://www.scrapy.org', + '/images/logo.png', + 'http://www.scrapy.org', + 'homepage', + 'http://www.scrapy.org', ]) def test_nested_load_item(self): @@ -514,9 +514,9 @@ class SubselectorLoaderTest(unittest.TestCase): assert item is nl1.item assert item is nl2.item - self.assertEqual(item['name'], [u'marta']) - self.assertEqual(item['url'], [u'http://www.scrapy.org']) - self.assertEqual(item['image'], [u'/images/logo.png']) + self.assertEqual(item['name'], ['marta']) + self.assertEqual(item['url'], ['http://www.scrapy.org']) + self.assertEqual(item['image'], ['/images/logo.png']) # Functions as processors diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index d0a59e8cd..eb14de14f 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -51,19 +51,19 @@ class BasicItemLoaderTest(unittest.TestCase): def test_load_item_using_default_loader(self): i = TestItem() - i['summary'] = u'lala' + i['summary'] = 'lala' il = ItemLoader(item=i) - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() assert item is i - self.assertEqual(item['summary'], [u'lala']) - self.assertEqual(item['name'], [u'marta']) + self.assertEqual(item['summary'], ['lala']) + self.assertEqual(item['name'], ['marta']) def test_load_item_using_custom_loader(self): il = TestItemLoader() - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() - self.assertEqual(item['name'], [u'Marta']) + self.assertEqual(item['name'], ['Marta']) def test_load_item_ignore_none_field_values(self): def validate_sku(value): @@ -76,23 +76,23 @@ class BasicItemLoaderTest(unittest.TestCase): price_out = Compose(TakeFirst(), float) sku_out = Compose(TakeFirst(), validate_sku) - valid_fragment = u'SKU: 1234' - invalid_fragment = u'SKU: not available' + valid_fragment = 'SKU: 1234' + invalid_fragment = 'SKU: not available' sku_re = 'SKU: (.+)' il = MyLoader(item={}) # Should not return "sku: None". il.add_value('sku', [invalid_fragment], re=sku_re) # Should not ignore empty values. - il.add_value('name', u'') - il.add_value('price', [u'0']) + il.add_value('name', '') + il.add_value('price', ['0']) self.assertEqual(il.load_item(), { - 'name': u'', + 'name': '', 'price': 0.0, }) il.replace_value('sku', [valid_fragment], re=sku_re) - self.assertEqual(il.load_item()['sku'], u'1234') + self.assertEqual(il.load_item()['sku'], '1234') def test_self_referencing_loader(self): class MyLoader(ItemLoader): @@ -117,19 +117,19 @@ class BasicItemLoaderTest(unittest.TestCase): def test_add_value(self): il = TestItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'Marta']) - self.assertEqual(il.get_output_value('name'), [u'Marta']) - il.add_value('name', u'pepe') - self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe']) - self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe']) + il.add_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['Marta']) + self.assertEqual(il.get_output_value('name'), ['Marta']) + il.add_value('name', 'pepe') + self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe']) + self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe']) # test add object value il.add_value('summary', {'key': 1}) self.assertEqual(il.get_collected_values('summary'), [{'key': 1}]) - il.add_value(None, u'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim']) + il.add_value(None, 'Jim', lambda x: {'name': x}) + self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim']) def test_add_zero(self): il = NameItemLoader() @@ -138,49 +138,49 @@ class BasicItemLoaderTest(unittest.TestCase): def test_replace_value(self): il = TestItemLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'Marta']) - self.assertEqual(il.get_output_value('name'), [u'Marta']) - il.replace_value('name', u'pepe') - self.assertEqual(il.get_collected_values('name'), [u'Pepe']) - self.assertEqual(il.get_output_value('name'), [u'Pepe']) + il.replace_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['Marta']) + self.assertEqual(il.get_output_value('name'), ['Marta']) + il.replace_value('name', 'pepe') + self.assertEqual(il.get_collected_values('name'), ['Pepe']) + self.assertEqual(il.get_output_value('name'), ['Pepe']) - il.replace_value(None, u'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), [u'Jim']) + il.replace_value(None, 'Jim', lambda x: {'name': x}) + self.assertEqual(il.get_collected_values('name'), ['Jim']) def test_get_value(self): il = NameItemLoader() - self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper)) - self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$')) - self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')) + self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper)) + self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$')) + self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')) - il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$') - self.assertEqual([u'foo'], il.get_collected_values('name')) - il.replace_value('name', u'name:bar', re=u'name:(.*)$') - self.assertEqual([u'bar'], il.get_collected_values('name')) + il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$') + self.assertEqual(['foo'], il.get_collected_values('name')) + il.replace_value('name', 'name:bar', re='name:(.*)$') + self.assertEqual(['bar'], il.get_collected_values('name')) def test_iter_on_input_processor_input(self): class NameFirstItemLoader(NameItemLoader): name_in = TakeFirst() il = NameFirstItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.add_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.add_value('name', [u'marta', u'jose']) - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.add_value('name', ['marta', 'jose']) + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.replace_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.replace_value('name', [u'marta', u'jose']) - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.replace_value('name', ['marta', 'jose']) + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.add_value('name', u'marta') - il.add_value('name', [u'jose', u'pedro']) - self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose']) + il.add_value('name', 'marta') + il.add_value('name', ['jose', 'pedro']) + self.assertEqual(il.get_collected_values('name'), ['marta', 'jose']) def test_map_compose_filter(self): def filter_world(x): @@ -195,87 +195,87 @@ class BasicItemLoaderTest(unittest.TestCase): name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) il = TestItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Mart']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['Mart']) item = il.load_item() - self.assertEqual(item['name'], [u'Mart']) + self.assertEqual(item['name'], ['Mart']) def test_default_input_processor(self): il = DefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mart']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['mart']) def test_inherited_default_input_processor(self): class InheritDefaultedItemLoader(DefaultedItemLoader): pass il = InheritDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mart']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['mart']) def test_input_processor_inheritance(self): class ChildItemLoader(TestItemLoader): url_in = MapCompose(lambda v: v.lower()) il = ChildItemLoader() - il.add_value('url', u'HTTP://scrapy.ORG') - self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org']) - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Marta']) + il.add_value('url', 'HTTP://scrapy.ORG') + self.assertEqual(il.get_output_value('url'), ['http://scrapy.org']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['Marta']) class ChildChildItemLoader(ChildItemLoader): url_in = MapCompose(lambda v: v.upper()) summary_in = MapCompose(lambda v: v) il = ChildChildItemLoader() - il.add_value('url', u'http://scrapy.org') - self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG']) - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Marta']) + il.add_value('url', 'http://scrapy.org') + self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['Marta']) def test_empty_map_compose(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): name_in = MapCompose() il = IdentityDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'marta']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['marta']) def test_identity_input_processor(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): name_in = Identity() il = IdentityDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'marta']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['marta']) def test_extend_custom_input_processors(self): class ChildItemLoader(TestItemLoader): name_in = MapCompose(TestItemLoader.name_in, str.swapcase) il = ChildItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mARTA']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['mARTA']) def test_extend_default_input_processors(self): class ChildDefaultedItemLoader(DefaultedItemLoader): name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase) il = ChildDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'MART']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['MART']) def test_output_processor_using_function(self): il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) class TakeFirstItemLoader(TestItemLoader): - name_out = u" ".join + name_out = " ".join il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar Ta') + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), 'Mar Ta') def test_output_processor_error(self): class TestItemLoader(ItemLoader): @@ -283,9 +283,9 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = MapCompose(float) il = TestItemLoader() - il.add_value('name', [u'$10']) + il.add_value('name', ['$10']) try: - float(u'$10') + float('$10') except Exception as e: expected_exc_str = str(e) @@ -303,53 +303,53 @@ class BasicItemLoaderTest(unittest.TestCase): def test_output_processor_using_classes(self): il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) class TakeFirstItemLoader(TestItemLoader): name_out = Join() il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar Ta') + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), 'Mar Ta') class TakeFirstItemLoader(TestItemLoader): name_out = Join("
    ") il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar
    Ta') + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), 'Mar
    Ta') def test_default_output_processor(self): il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) class LalaItemLoader(TestItemLoader): default_output_processor = Identity() il = LalaItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) def test_loader_context_on_declaration(self): class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args, key=u'val') + url_in = MapCompose(processor_with_args, key='val') il = ChildItemLoader() - il.add_value('url', u'text') + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['val']) def test_loader_context_on_instantiation(self): class ChildItemLoader(TestItemLoader): url_in = MapCompose(processor_with_args) - il = ChildItemLoader(key=u'val') - il.add_value('url', u'text') + il = ChildItemLoader(key='val') + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['val']) def test_loader_context_on_assign(self): @@ -357,10 +357,10 @@ class BasicItemLoaderTest(unittest.TestCase): url_in = MapCompose(processor_with_args) il = ChildItemLoader() - il.context['key'] = u'val' - il.add_value('url', u'text') + il.context['key'] = 'val' + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['val']) def test_item_passed_to_input_processor_functions(self): @@ -372,9 +372,9 @@ class BasicItemLoaderTest(unittest.TestCase): it = TestItem(name='marta') il = ChildItemLoader(item=it) - il.add_value('url', u'text') + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['marta']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['marta']) def test_compose_processor(self): @@ -382,10 +382,10 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) il = TestItemLoader() - il.add_value('name', [u'marta', u'other']) - self.assertEqual(il.get_output_value('name'), u'Mart') + il.add_value('name', ['marta', 'other']) + self.assertEqual(il.get_output_value('name'), 'Mart') item = il.load_item() - self.assertEqual(item['name'], u'Mart') + self.assertEqual(item['name'], 'Mart') def test_partial_processor(self): def join(values, sep=None, loader_context=None, ignored=None): @@ -402,13 +402,13 @@ class BasicItemLoaderTest(unittest.TestCase): summary_out = Compose(partial(join, ignored='foo')) il = TestItemLoader() - il.add_value('name', [u'rabbit', u'hole']) - il.add_value('url', [u'rabbit', u'hole']) - il.add_value('summary', [u'rabbit', u'hole']) + il.add_value('name', ['rabbit', 'hole']) + il.add_value('url', ['rabbit', 'hole']) + il.add_value('summary', ['rabbit', 'hole']) item = il.load_item() - self.assertEqual(item['name'], u'rabbit+hole') - self.assertEqual(item['url'], u'rabbit.hole') - self.assertEqual(item['summary'], u'rabbithole') + self.assertEqual(item['name'], 'rabbit+hole') + self.assertEqual(item['url'], 'rabbit.hole') + self.assertEqual(item['summary'], 'rabbithole') def test_error_input_processor(self): class TestItem(Item): @@ -420,7 +420,7 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() self.assertRaises(ValueError, il.add_value, 'name', - [u'marta', u'other']) + ['marta', 'other']) def test_error_output_processor(self): class TestItem(Item): @@ -431,7 +431,7 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = Compose(Join(), float) il = TestItemLoader() - il.add_value('name', u'marta') + il.add_value('name', 'marta') with self.assertRaises(ValueError): il.load_item() @@ -444,7 +444,7 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() self.assertRaises(ValueError, il.add_value, 'name', - [u'marta', u'other'], Compose(float)) + ['marta', 'other'], Compose(float)) class InitializationFromDictTest(unittest.TestCase): @@ -608,8 +608,8 @@ class ProcessorsTest(unittest.TestCase): def test_join(self): proc = Join() self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) - self.assertEqual(proc(['', 'hello', 'world']), u' hello world') - self.assertEqual(proc(['hello', 'world']), u'hello world') + self.assertEqual(proc(['', 'hello', 'world']), ' hello world') + self.assertEqual(proc(['hello', 'world']), 'hello world') self.assertIsInstance(proc(['hello', 'world']), str) def test_compose(self): @@ -626,8 +626,8 @@ class ProcessorsTest(unittest.TestCase): def filter_world(x): return None if x == 'world' else x proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), - [u'HELLO', u'THIS', u'IS', u'SCRAPY']) + self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), + ['HELLO', 'THIS', 'IS', 'SCRAPY']) proc = MapCompose(filter_world, str.upper) self.assertEqual(proc(None), []) proc = MapCompose(filter_world, str.upper) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 7064337ad..b771e7d79 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -56,13 +56,13 @@ class LogFormatterTestCase(unittest.TestCase): def test_dropped(self): item = {} - exception = Exception(u"\u2018") + exception = Exception("\u2018") response = Response("http://www.example.com") logkws = self.formatter.dropped(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, [u"Dropped: \u2018", '{}']) + self.assertEqual(lines, ["Dropped: \u2018", '{}']) def test_item_error(self): # In practice, the complete traceback is shown by passing the @@ -72,7 +72,7 @@ class LogFormatterTestCase(unittest.TestCase): response = Response("http://www.example.com") logkws = self.formatter.item_error(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, u"Error processing {'key': 'value'}") + self.assertEqual(logline, "Error processing {'key': 'value'}") def test_spider_error(self): # In practice, the complete traceback is shown by passing the @@ -107,20 +107,20 @@ class LogFormatterTestCase(unittest.TestCase): def test_scraped(self): item = CustomItem() - item['name'] = u'\xa3' + item['name'] = '\xa3' response = Response("http://www.example.com") logkws = self.formatter.scraped(item, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3']) + self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3']) class LogFormatterSubclass(LogFormatter): def crawled(self, request, response, spider): kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider) CRAWLEDMSG = ( - u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" + "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" ) log_args = kwargs['args'] log_args['flags'] = str(request.flags) diff --git a/tests/test_mail.py b/tests/test_mail.py index 53dbc0686..9b248fbfa 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -73,8 +73,8 @@ class MailSenderTest(unittest.TestCase): self.catched_msg = dict(**kwargs) def test_send_utf8(self): - subject = u'sübjèçt' - body = u'bödÿ-àéïöñß' + subject = 'sübjèçt' + body = 'bödÿ-àéïöñß' mailsender = MailSender(debug=True) mailsender.send(to=['test@scrapy.org'], subject=subject, body=body, charset='utf-8', _callback=self._catch_mail_sent) @@ -90,8 +90,8 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"') def test_send_attach_utf8(self): - subject = u'sübjèçt' - body = u'bödÿ-àéïöñß' + subject = 'sübjèçt' + body = 'bödÿ-àéïöñß' attach = BytesIO() attach.write(body.encode('utf-8')) attach.seek(0) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index dd19a69d5..a175f88ca 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -23,11 +23,11 @@ class ResponseTypesTest(unittest.TestCase): mappings = [ (b'attachment; filename="data.xml"', XmlResponse), (b'attachment; filename=data.xml', XmlResponse), - (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response), - (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response), - (u'attachment;filename=data高.doc'.encode('gbk'), Response), - (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse), - (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse), + ('attachment;filename=data£.tar.gz'.encode('utf-8'), Response), + ('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response), + ('attachment;filename=data高.doc'.encode('gbk'), Response), + ('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse), + ('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse), ] for source, cls in mappings: diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 24aaaf7ec..9d8c201dd 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -93,7 +93,7 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://site.local/disallowed", "*")) def test_unicode_url_and_useragent(self): - robotstxt_robotstxt_body = u""" + robotstxt_robotstxt_body = """ User-Agent: * Disallow: /admin/ Disallow: /static/ @@ -107,11 +107,11 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://site.local/", "*")) self.assertFalse(rp.allowed("https://site.local/admin/", "*")) self.assertFalse(rp.allowed("https://site.local/static/", "*")) - self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt")) + self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt")) self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*")) - self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*")) + self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*")) self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*")) - self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt")) + self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt")) class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): diff --git a/tests/test_selector.py b/tests/test_selector.py index 00e663c11..62036ad8c 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -25,19 +25,19 @@ class SelectorTestCase(unittest.TestCase): ) self.assertEqual( [x.get() for x in sel.xpath("//input[@name='a']/@name")], - [u'a'] + ['a'] ) self.assertEqual( [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], - [u'12.0'] + ['12.0'] ) self.assertEqual( sel.xpath("concat('xpath', 'rules')").getall(), - [u'xpathrules'] + ['xpathrules'] ) self.assertEqual( [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], - [u'12'] + ['12'] ) def test_root_base_url(self): @@ -52,30 +52,30 @@ class SelectorTestCase(unittest.TestCase): sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'xml') self.assertEqual(sel.xpath("//div").getall(), - [u'

    Hello

    ']) + ['

    Hello

    ']) sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'html') self.assertEqual(sel.xpath("//div").getall(), - [u'

    Hello

    ']) + ['

    Hello

    ']) def test_http_header_encoding_precedence(self): - # u'\xa3' = pound symbol in unicode - # u'\xc2\xa3' = pound symbol in utf-8 - # u'\xa3' = pound symbol in latin-1 (iso-8859-1) + # '\xa3' = pound symbol in unicode + # '\xc2\xa3' = pound symbol in utf-8 + # '\xa3' = pound symbol in latin-1 (iso-8859-1) - meta = u'' - head = u'' + meta + u'' - body_content = u'\xa3' - body = u'' + body_content + u'' - html = u'' + head + body + u'' + meta = '' + head = '' + meta + '' + body_content = '\xa3' + body = '' + body_content + '' + html = '' + head + body + '' encoding = 'utf-8' html_utf8 = html.encode(encoding) headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3']) + self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3']) def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence @@ -92,4 +92,4 @@ class SelectorTestCase(unittest.TestCase): def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, 'received both response and text'): - Selector(TextResponse(url='http://example.com', body=b''), text=u'') + Selector(TextResponse(url='http://example.com', body=b''), text='') diff --git a/tests/test_spider.py b/tests/test_spider.py index 83c10a3c3..78157a9b9 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -153,13 +153,13 @@ class XMLFeedSpiderTest(SpiderTest): output = list(spider._parse(response)) self.assertEqual(len(output), 2, iterator) self.assertEqual(output, [ - {'loc': [u'http://www.example.com/Special-Offers.html'], - 'updated': [u'2009-08-16'], - 'custom': [u'fuu'], - 'other': [u'bar']}, + {'loc': ['http://www.example.com/Special-Offers.html'], + 'updated': ['2009-08-16'], + 'custom': ['fuu'], + 'other': ['bar']}, {'loc': [], - 'updated': [u'2009-08-16'], - 'other': [u'foo'], + 'updated': ['2009-08-16'], + 'other': ['foo'], 'custom': []}, ], iterator) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index d17bb2cbc..298178f08 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -54,7 +54,7 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 - body = u""" + body = """ <þingflokkar> <þingflokkur id="26"> @@ -97,15 +97,15 @@ class XmliterTestCase(unittest.TestCase): XmlResponse(url="http://example.com", body=body, encoding='utf-8'), ): attrs = [] - for x in self.xmliter(r, u'þingflokkur'): + for x in self.xmliter(r, 'þingflokkur'): attrs.append((x.attrib['id'], - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(), - x.xpath(u'./tímabil/fyrstaþing/text()').getall())) + x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(), + x.xpath('./tímabil/fyrstaþing/text()').getall())) self.assertEqual(attrs, - [(u'26', [u'-'], [u'80']), - (u'21', [u'Ab'], [u'76']), - (u'27', [u'A'], [u'27'])]) + [('26', ['-'], ['80']), + ('21', ['Ab'], ['76']), + ('27', ['A'], ['27'])]) def test_xmliter_text(self): body = ( @@ -114,7 +114,7 @@ class XmliterTestCase(unittest.TestCase): ) self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')], - [[u'one'], [u'two']]) + [['one'], ['two']]) def test_xmliter_namespaces(self): body = b""" @@ -179,7 +179,7 @@ class XmliterTestCase(unittest.TestCase): response = XmlResponse('http://www.example.com', body=body) self.assertEqual( next(self.xmliter(response, 'item')).get(), - u'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' + 'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' ) @@ -265,10 +265,10 @@ class UtilsCsvTestCase(unittest.TestCase): result = [row for row in csv] self.assertEqual(result, - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: @@ -281,10 +281,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter='\t') self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_quotechar(self): body1 = get_testdata('feeds', 'feed-sample6.csv') @@ -294,19 +294,19 @@ class UtilsCsvTestCase(unittest.TestCase): csv1 = csviter(response1, quotechar="'") self.assertEqual([row for row in csv1], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) response2 = TextResponse(url="http://example.com/", body=body2) csv2 = csviter(response2, delimiter="|", quotechar="'") self.assertEqual([row for row in csv2], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_wrong_quotechar(self): body = get_testdata('feeds', 'feed-sample6.csv') @@ -314,10 +314,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual([row for row in csv], - [{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"}, - {u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"}, - {u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"}, - {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}]) + [{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, + {"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"}, + {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"}, + {"'id'": "4", "'name'": "'empty'", "'value'": ""}]) def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') @@ -325,10 +325,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter='\t') self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_headers(self): sample = get_testdata('feeds', 'feed-sample3.csv').splitlines() @@ -338,10 +338,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, headers=[h.decode('utf-8') for h in headers]) self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': 'foo\nbar'}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_falserow(self): body = get_testdata('feeds', 'feed-sample3.csv') @@ -351,10 +351,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_exception(self): body = get_testdata('feeds', 'feed-sample3.csv') @@ -377,8 +377,8 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual( list(csv), [ - {u'id': u'1', u'name': u'latin1', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}, + {'id': '1', 'name': 'latin1', 'value': 'test'}, + {'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'}, ] ) @@ -387,8 +387,8 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual( list(csv), [ - {u'id': u'1', u'name': u'cp852', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}, + {'id': '1', 'name': 'cp852', 'value': 'test'}, + {'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}, ] ) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index ebce3c079..3f93f509e 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -34,13 +34,13 @@ class MutableChainTest(unittest.TestCase): class ToUnicodeTest(unittest.TestCase): def test_converting_an_utf8_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e') + self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e') def test_converting_a_latin_1_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e') + self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e') def test_converting_a_unicode_to_unicode_should_return_the_same_object(self): - self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e') + self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e') def test_converting_a_strange_object_should_raise_TypeError(self): self.assertRaises(TypeError, to_unicode, 423) @@ -48,16 +48,16 @@ class ToUnicodeTest(unittest.TestCase): def test_errors_argument(self): self.assertEqual( to_unicode(b'a\xedb', 'utf-8', errors='replace'), - u'a\ufffdb' + 'a\ufffdb' ) class ToBytesTest(unittest.TestCase): def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self): - self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49') + self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49') def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self): - self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49') + self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49') def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self): self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e') @@ -67,7 +67,7 @@ class ToBytesTest(unittest.TestCase): def test_errors_argument(self): self.assertEqual( - to_bytes(u'a\ufffdb', 'latin-1', errors='replace'), + to_bytes('a\ufffdb', 'latin-1', errors='replace'), b'a?b' ) @@ -96,7 +96,7 @@ class BinaryIsTextTest(unittest.TestCase): assert binary_is_text(b"hello") def test_utf_16_strings_contain_null_bytes(self): - assert binary_is_text(u"hello".encode('utf-16')) + assert binary_is_text("hello".encode('utf-16')) def test_one_with_encoding(self): assert binary_is_text(b"
    Price \xa3
    ") diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 450e4bdca..de94ec960 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -22,7 +22,7 @@ class RequestSerializationTest(unittest.TestCase): method="POST", body=b"some body", headers={'content-encoding': 'text/html; charset=latin-1'}, - cookies={'currency': u'руб'}, + cookies={'currency': 'руб'}, encoding='latin-1', priority=20, meta={'a': 'b'}, diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 5a52dd695..5ff2e41ef 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -19,8 +19,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): def test_simple_render(self): context = dict(project_name='proj', name='spi', classname='TheSpider') - template = u'from ${project_name}.spiders.${name} import ${classname}' - rendered = u'from proj.spiders.spi import TheSpider' + template = 'from ${project_name}.spiders.${name} import ${classname}' + rendered = 'from proj.spiders.spi import TheSpider' template_path = os.path.join(self.tmp_path, 'templ.py.tmpl') render_path = os.path.join(self.tmp_path, 'templ.py') From 6f4ccec5675c00b0ec7877a0b1fcc234d5983490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Jul 2020 14:03:14 +0200 Subject: [PATCH 213/568] Cover our deprecation policy in the documentation --- docs/contributing.rst | 5 +++++ docs/versioning.rst | 22 ++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 7b901dd00..525ad3497 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -108,6 +108,11 @@ Well-written patches should: tox -e docs-coverage +* if you are removing deprecated code, first make sure that at least 1 year + (12 months) has passed since the release that introduced the deprecation. + See :ref:`deprecation-policy`. + + .. _submitting-patches: Submitting patches diff --git a/docs/versioning.rst b/docs/versioning.rst index 227085f02..57643ea9a 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -1,7 +1,7 @@ .. _versioning: ============================ -Versioning and API Stability +Versioning and API stability ============================ Versioning @@ -34,7 +34,7 @@ For example: production) -API Stability +API stability ============= API stability was one of the major goals for the *1.0* release. @@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the same way. +.. _deprecation-policy: + +Deprecation policy +================== + +We aim to maintain support for deprecated Scrapy features for at least 1 year. + +For example, if a feature is deprecated in a Scrapy version released on +June 15th 2020, that feature should continue to work in versions released on +June 14th 2021 or before that. + +Any new Scrapy release after a year *may* remove support for that deprecated +feature. + +All deprecated features removed in a Scrapy release are explicitly mentioned in +the :ref:`release notes `. + + .. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases From d707f8b5d94856ba13bd8e76acb1efb8de377f9c Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 30 Jul 2020 18:06:21 +0530 Subject: [PATCH 214/568] docs: mention H2DownloadHandler in settings.rst --- docs/topics/settings.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5178f272f..670b44f3c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -620,6 +620,13 @@ handler (without replacement), place this in your ``settings.py``:: 'ftp': None, } +The default https handler uses HTTP/1.x, to use HTTP/2.0 update :setting:`DOWNLOAD_HANDLERS` +as:: + + DOWNLOAD_HANDLERS = { + 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', + } + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT @@ -697,6 +704,14 @@ Optionally, this can be set per-request basis by using the If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``, the ``ResponseFailed([_DataLoss])`` failure will be retried as usual. +.. warning:: + + This is ignored when :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` + is set as ``https`` download handler in :setting:`DOWNLOAD_HANDLERS`. In + case of data loss error the connection may be corrupted affecting other streams, + hence all streams return with the ``ResponseFailed([InvalidBodyLengthError])`` + failure. + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS From 3600582f56071dd9f40f31ed44e09294a78dc13f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 4 Aug 2020 20:05:56 +0200 Subject: [PATCH 215/568] Cover Scrapy 2.2.1 and 2.3 in the release notes (#4708) --- docs/news.rst | 133 ++++++++++++++++++++++++++++++++ docs/topics/commands.rst | 2 + docs/topics/developer-tools.rst | 6 +- docs/topics/feed-exports.rst | 36 ++++++++- scrapy/utils/curl.py | 3 +- 5 files changed, 176 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 80d130e4a..850b323ef 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,139 @@ Release notes ============= +.. _release-2.3.0: + +Scrapy 2.3.0 (2020-08-04) +------------------------- + +Highlights: + +* :ref:`Feed exports ` now support :ref:`Google Cloud + Storage ` as a storage backend + +* The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver + output items in batches of up to the specified number of items. + + It also serves as a workaround for :ref:`delayed file delivery + `, which causes Scrapy to only start item delivery + after the crawl has finished when using certain storage backends + (:ref:`S3 `, :ref:`FTP `, + and now :ref:`GCS `). + +* The base implementation of :ref:`item loaders ` has been + moved into a separate library, :doc:`itemloaders `, + allowing usage from outside Scrapy and a separate release schedule + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* Removed the following classes and their parent modules from + ``scrapy.linkextractors``: + + * ``htmlparser.HtmlParserLinkExtractor`` + * ``regex.RegexLinkExtractor`` + * ``sgml.BaseSgmlLinkExtractor`` + * ``sgml.SgmlLinkExtractor`` + + Use + :class:`LinkExtractor ` + instead (:issue:`4356`, :issue:`4679`) + + +Deprecations +~~~~~~~~~~~~ + +* The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated + (:issue:`4683`) + + +New features +~~~~~~~~~~~~ + +* :ref:`Feed exports ` support :ref:`Google Cloud + Storage ` (:issue:`685`, :issue:`3608`) + +* New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries + (:issue:`4250`, :issue:`4434`) + +* The :command:`parse` command now allows specifying an output file + (:issue:`4317`, :issue:`4377`) + +* :meth:`Request.from_curl ` and + :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support + ``--data-raw`` (:issue:`4612`) + +* A ``parse`` callback may now be used in built-in spider subclasses, such + as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`, + :issue:`781`, :issue:`4254` ) + + +Bug fixes +~~~~~~~~~ + +* Fixed the :ref:`CSV exporting ` of + :ref:`dataclass items ` and :ref:`attr.s items + ` (:issue:`4667`, :issue:`4668`) + +* :meth:`Request.from_curl ` and + :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request + method to ``POST`` when a request body is specified and no request method + is specified (:issue:`4612`) + +* The processing of ANSI escape sequences in enabled in Windows 10.0.14393 + and later, where it is required for colored output (:issue:`4393`, + :issue:`4403`) + + +Documentation +~~~~~~~~~~~~~ + +* Updated the `OpenSSL cipher list format`_ link in the documentation about + the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`) + +* Simplified the code example in :ref:`topics-loaders-dataclass` + (:issue:`4652`) + +.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* The base implementation of :ref:`item loaders ` has been + moved into :doc:`itemloaders ` (:issue:`4005`, + :issue:`4516`) + +* Fixed a silenced error in some scheduler tests (:issue:`4644`, + :issue:`4645`) + +* Renewed the localhost certificate used for SSL tests (:issue:`4650`) + +* Removed cookie-handling code specific to Python 2 (:issue:`4682`) + +* Stopped using Python 2 unicode literal syntax (:issue:`4704`) + +* Stopped using a backlash for line continuation (:issue:`4673`) + +* Removed unneeded entries from the MyPy exception list (:issue:`4690`) + +* Automated tests now pass on Windows as part of our continuous integration + system (:issue:`4458`) + +* Automated tests now pass on the latest PyPy version for supported Python + versions in our continuous integration system (:issue:`4504`) + + +.. _release-2.2.1: + +Scrapy 2.2.1 (2020-07-17) +------------------------- + +* The :command:`startproject` command no longer makes unintended changes to + the permissions of files in the destination folder, such as removing + execution permissions (:issue:`4662`, :issue:`4666`) + + .. _release-2.2.0: Scrapy 2.2.0 (2020-06-24) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 4fce51abc..9638a2322 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -493,6 +493,8 @@ Supported options: * ``--output`` or ``-o``: dump scraped items to a file + .. versionadded:: 2.3 + .. skip: start Usage example:: diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 4e87a00f2..101aa159c 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -289,8 +289,10 @@ request:: "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'") Alternatively, if you want to know the arguments needed to recreate that -request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs` -function to get a dictionary with the equivalent arguments. +request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs` +function to get a dictionary with the equivalent arguments: + +.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs Note that to translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index dd4eb3c61..37b7096f6 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -100,6 +100,7 @@ The storages backends supported out of the box are: * :ref:`topics-feed-storage-fs` * :ref:`topics-feed-storage-ftp` * :ref:`topics-feed-storage-s3` (requires botocore_) + * :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) * :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are @@ -169,6 +170,9 @@ FTP supports two different connection modes: `active or passive mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. +This storage backend uses :ref:`delayed file delivery `. + + .. _topics-feed-storage-s3: S3 @@ -194,11 +198,16 @@ You can also define a custom ACL for exported feeds using this setting: * :setting:`FEED_STORAGE_S3_ACL` +This storage backend uses :ref:`delayed file delivery `. + + .. _topics-feed-storage-gcs: Google Cloud Storage (GCS) -------------------------- +.. versionadded:: 2.3 + The feeds are stored on `Google Cloud Storage`_. * URI scheme: ``gs`` @@ -206,7 +215,7 @@ The feeds are stored on `Google Cloud Storage`_. * ``gs://mybucket/path/to/export.csv`` - * Required external libraries: `google-cloud-storage `_. + * Required external libraries: `google-cloud-storage`_. For more information about authentication, please refer to `Google Cloud documentation `_. @@ -215,6 +224,11 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following * :setting:`FEED_STORAGE_GCS_ACL` * :setting:`GCS_PROJECT_ID` +This storage backend uses :ref:`delayed file delivery `. + +.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python + + .. _topics-feed-storage-stdout: Standard output @@ -227,6 +241,26 @@ The feeds are written to the standard output of the Scrapy process. * Required external libraries: none +.. _delayed-file-delivery: + +Delayed file delivery +--------------------- + +As indicated above, some of the described storage backends use delayed file +delivery. + +These storage backends do not upload items to the feed URI as those items are +scraped. Instead, Scrapy writes items into a temporary local file, and only +once all the file contents have been written (i.e. at the end of the crawl) is +that file uploaded to the feed URI. + +If you want item delivery to start earlier when using one of these storage +backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items +in multiple files, with the specified maximum item count per file. That way, as +soon as a file reaches the maximum item count, that file is delivered to the +feed URI, allowing item delivery to start way before the end of the crawl. + + Settings ======== diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index aa681522f..9c0efcec4 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -39,7 +39,8 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): :param str curl_command: string containing the curl command :param bool ignore_unknown_options: If true, only a warning is emitted when - cURL options are unknown. Otherwise raises an error. (default: True) + cURL options are unknown. Otherwise + raises an error. (default: True) :return: dictionary of Request kwargs """ From 1278e76d9093b1c5c9ec810768d1066772a8a134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 4 Aug 2020 20:07:02 +0200 Subject: [PATCH 216/568] =?UTF-8?q?Bump=20version:=202.2.0=20=E2=86=92=202?= =?UTF-8?q?.3.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8d4d74bc5..3c1c8f891 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.2.0 +current_version = 2.3.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index ccbccc3dc..276cbf9e2 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.2.0 +2.3.0 From 4ee538e44b2650c49054b1f7f4c87ac70350471a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 4 Aug 2020 20:34:11 +0200 Subject: [PATCH 217/568] Update unicode references from Python 2 times in the documentation (#4703) --- docs/topics/exporters.rst | 8 ++------ docs/topics/loaders.rst | 4 ++-- docs/topics/request-response.rst | 30 ++++++++++++++++-------------- docs/topics/selectors.rst | 7 ++++--- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index e5c99e5b1..8c84b85fc 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -166,8 +166,7 @@ BaseItemExporter By default, this method looks for a serializer :ref:`declared in the item field ` and returns the result of applying that serializer to the value. If no serializer is found, it returns the - value unchanged except for ``unicode`` values which are encoded to - ``str`` using the encoding declared in the :attr:`encoding` attribute. + value unchanged. :param field: the field being serialized. If the source :ref:`item object ` does not define field metadata, *field* is an empty @@ -217,10 +216,7 @@ BaseItemExporter .. attribute:: encoding - The encoding that will be used to encode unicode values. This only - affects unicode values (which are always serialized to str using this - encoding). Other value types are passed unchanged to the specific - serialization library. + The output character encoding. .. attribute:: indent diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 29d9c5805..c0f534493 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -193,10 +193,10 @@ Item Loaders are declared using a class definition syntax. Here is an example:: default_output_processor = TakeFirst() - name_in = MapCompose(unicode.title) + name_in = MapCompose(str.title) name_out = Join() - price_in = MapCompose(unicode.strip) + price_in = MapCompose(str.strip) # ... diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index fbd8e4b73..1dffd1d55 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -51,12 +51,12 @@ Request objects given, the dict passed in this parameter will be shallow copied. :type meta: dict - :param body: the request body. If a ``unicode`` is passed, then it's encoded to - ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If - ``body`` is not given, an empty string is stored. Regardless of the - type of this argument, the final value stored will be a ``str`` (never - ``unicode`` or ``None``). - :type body: str or unicode + :param body: the request body. If a string is passed, then it's encoded as + bytes using the ``encoding`` passed (which defaults to ``utf-8``). If + ``body`` is not given, an empty bytes object is stored. Regardless of the + type of this argument, the final value stored will be a bytes object + (never a string or ``None``). + :type body: bytes or str :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If @@ -106,7 +106,7 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the - body to ``str`` (if given as ``unicode``). + body to bytes (if given as a string). :type encoding: string :param priority: the priority of this request (defaults to ``0``). @@ -721,7 +721,7 @@ Response objects .. attribute:: Response.body The body of this Response. Keep in mind that Response.body - is always a bytes object. If you want the unicode version use + is always a bytes object. If you want the string version use :attr:`TextResponse.text` (only available in :class:`TextResponse` and subclasses). @@ -842,9 +842,9 @@ TextResponse objects is the same as for the :class:`Response` class and is not documented here. :param encoding: is a string which contains the encoding to use for this - response. If you create a :class:`TextResponse` object with a unicode + response. If you create a :class:`TextResponse` object with a string as body, it will be encoded using this encoding (remember the body attribute - is always a string). If ``encoding`` is ``None`` (default value), the + is always a bytes object). If ``encoding`` is ``None`` (default value), the encoding will be looked up in the response headers and body instead. :type encoding: string @@ -853,7 +853,7 @@ TextResponse objects .. attribute:: TextResponse.text - Response body, as unicode. + Response body, as a string. The same as ``response.body.decode(response.encoding)``, but the result is cached after the first call, so you can access @@ -861,9 +861,11 @@ TextResponse objects .. note:: - ``unicode(response.body)`` is not a correct way to convert response - body to unicode: you would be using the system default encoding - (typically ``ascii``) instead of the response encoding. + ``str(response.body)`` is not a correct way to convert the response + body into a string: + + >>> str(b'body') + "b'body'" .. attribute:: TextResponse.encoding diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 5014df6ac..9e2c6ba42 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -64,7 +64,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``: Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class constructed by passing either :class:`~scrapy.http.TextResponse` object or -markup as an unicode string (in ``text`` argument). +markup as a string (in ``text`` argument). + Usually there is no need to construct Scrapy selectors manually: ``response`` object is available in Spider callbacks, so in most cases it is more convenient to use ``response.css()`` and ``response.xpath()`` @@ -383,7 +384,7 @@ Using selectors with regular expressions :class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting data using regular expressions. However, unlike using ``.xpath()`` or -``.css()`` methods, ``.re()`` returns a list of unicode strings. So you +``.css()`` methods, ``.re()`` returns a list of strings. So you can't construct nested ``.re()`` calls. Here's an example used to extract image names from the :ref:`HTML code @@ -989,7 +990,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this:: sel.xpath("//h1") 2. Extract the text of all ``

    `` elements from an HTML response body, - returning a list of unicode strings:: + returning a list of strings:: sel.xpath("//h1").getall() # this includes the h1 tag sel.xpath("//h1/text()").getall() # this excludes the h1 tag From 336f19f5cc6edd0392c77f38b857d3e40bf565da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= Date: Tue, 4 Aug 2020 20:42:01 +0200 Subject: [PATCH 218/568] Change super syntax (#4707) --- extras/qpsclient.py | 2 +- scrapy/commands/view.py | 2 +- scrapy/contracts/default.py | 2 +- scrapy/core/downloader/contextfactory.py | 4 ++-- scrapy/core/downloader/handlers/http11.py | 10 +++++----- scrapy/core/downloader/tls.py | 2 +- scrapy/core/spidermw.py | 2 +- scrapy/crawler.py | 2 +- scrapy/downloadermiddlewares/redirect.py | 2 +- scrapy/exceptions.py | 4 ++-- scrapy/exporters.py | 2 +- scrapy/http/headers.py | 8 ++++---- scrapy/http/request/form.py | 2 +- scrapy/http/request/json_request.py | 4 ++-- scrapy/http/request/rpc.py | 2 +- scrapy/http/response/text.py | 10 +++++----- scrapy/item.py | 10 +++++----- scrapy/linkextractors/__init__.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/pipelines/files.py | 2 +- scrapy/pipelines/images.py | 3 +-- scrapy/resolver.py | 10 +++++----- scrapy/selector/unified.py | 2 +- scrapy/settings/__init__.py | 2 +- scrapy/spidermiddlewares/httperror.py | 2 +- scrapy/spiders/crawl.py | 4 ++-- scrapy/spiders/init.py | 2 +- scrapy/spiders/sitemap.py | 2 +- scrapy/squeues.py | 12 ++++++------ scrapy/statscollectors.py | 2 +- scrapy/utils/datatypes.py | 14 +++++++------- scrapy/utils/deprecate.py | 8 ++++---- scrapy/utils/log.py | 2 +- scrapy/utils/serialize.py | 2 +- scrapy/utils/testsite.py | 4 ++-- tests/spiders.py | 20 ++++++++++---------- tests/test_command_parse.py | 2 +- tests/test_commands.py | 4 ++-- tests/test_contracts.py | 2 +- tests/test_downloader_handlers.py | 4 ++-- tests/test_downloadermiddleware_httpcache.py | 4 ++-- tests/test_downloadermiddleware_robotstxt.py | 4 ++-- tests/test_exporters.py | 2 +- tests/test_http_request.py | 4 ++-- tests/test_http_response.py | 2 +- tests/test_item.py | 2 +- tests/test_linkextractors.py | 2 +- tests/test_loader.py | 2 +- tests/test_loader_deprecated.py | 2 +- tests/test_logformatter.py | 2 +- tests/test_middleware.py | 2 +- tests/test_pipeline_media.py | 12 ++++++------ tests/test_request_left.py | 2 +- tests/test_robotstxt_interface.py | 8 ++++---- tests/test_scheduler.py | 4 ++-- tests/test_spidermiddleware_httperror.py | 2 +- 56 files changed, 117 insertions(+), 118 deletions(-) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 7554f7eec..fe1f96cbb 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -27,7 +27,7 @@ class QPSSpider(Spider): slots = 1 def __init__(self, *a, **kw): - super(QPSSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) if self.qps is not None: self.qps = float(self.qps) self.download_delay = 1 / self.qps diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 908bee966..c8f873334 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -11,7 +11,7 @@ class Command(fetch.Command): return "Fetch a URL using the Scrapy downloader and show its contents in a browser" def add_options(self, parser): - super(Command, self).add_options(parser) + super().add_options(parser) parser.remove_option("--headers") def _print_response(self, response, opts): diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 34f0d36d4..cfdcc7c25 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -56,7 +56,7 @@ class ReturnsContract(Contract): } def __init__(self, *args, **kwargs): - super(ReturnsContract, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if len(self.args) not in [1, 2, 3]: raise ValueError( diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 452242d47..8a7d656a1 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): """ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs): - super(ScrapyClientContextFactory, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._ssl_method = method self.tls_verbose_logging = tls_verbose_logging if tls_ciphers: @@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133) # # * getattr() for `_ssl_method` attribute for context factories - # not calling super(..., self).__init__ + # not calling super().__init__ return CertificateOptions( verify=False, method=getattr(self, 'method', getattr(self, '_ssl_method', None)), diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 22c9ac520..fb04d1fb7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf - super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) + super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) self._tunnelReadyDeferred = defer.Deferred() self._tunneledHost = host self._tunneledPort = port @@ -178,7 +178,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def connect(self, protocolFactory): self._protocolFactory = protocolFactory - connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory) + connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) connectDeferred.addErrback(self.connectFailed) return self._tunnelReadyDeferred @@ -215,7 +215,7 @@ class TunnelingAgent(Agent): def __init__(self, reactor, proxyConf, contextFactory=None, connectTimeout=None, bindAddress=None, pool=None): - super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) + super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) self._proxyConf = proxyConf self._contextFactory = contextFactory @@ -235,7 +235,7 @@ class TunnelingAgent(Agent): # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy key = key + self._proxyConf - return super(TunnelingAgent, self)._requestWithEndpoint( + return super()._requestWithEndpoint( key=key, endpoint=endpoint, method=method, @@ -249,7 +249,7 @@ class TunnelingAgent(Agent): class ScrapyProxyAgent(Agent): def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None): - super(ScrapyProxyAgent, self).__init__( + super().__init__( reactor=reactor, connectTimeout=connectTimeout, bindAddress=bindAddress, diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index e43a3c83e..d9f3750d5 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions): """ def __init__(self, hostname, ctx, verbose_logging=False): - super(ScrapyClientTLSOptions, self).__init__(hostname, ctx) + super().__init__(hostname, ctx) self.verbose_logging = verbose_logging def _identityVerifyingInfoCallback(self, connection, where, ret): diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 35264a92b..5a99b96be 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) def _add_middleware(self, mw): - super(SpiderMiddlewareManager, self)._add_middleware(mw) + super()._add_middleware(mw) if hasattr(mw, 'process_spider_input'): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_start_requests'): diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 6f43771e2..48f19424c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner): """ def __init__(self, settings=None, install_root_handler=True): - super(CrawlerProcess, self).__init__(settings) + super().__init__(settings) install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 366d60dcb..4053fecc5 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -92,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): enabled_setting = 'METAREFRESH_ENABLED' def __init__(self, settings): - super(MetaRefreshMiddleware, self).__init__(settings) + super().__init__(settings) self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS') self._maxdelay = settings.getint('METAREFRESH_MAXDELAY') diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 45f152321..0c410f035 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -37,7 +37,7 @@ class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" def __init__(self, reason='cancelled'): - super(CloseSpider, self).__init__() + super().__init__() self.reason = reason @@ -74,7 +74,7 @@ class UsageError(Exception): def __init__(self, *a, **kw): self.print_help = kw.pop('print_help', True) - super(UsageError, self).__init__(*a, **kw) + super().__init__(*a, **kw) class ScrapyDeprecationWarning(Warning): diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 0aba1c904..95518b3ac 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -301,7 +301,7 @@ class PythonItemExporter(BaseItemExporter): def _configure(self, options, dont_fail=False): self.binary = options.pop('binary', True) - super(PythonItemExporter, self)._configure(options, dont_fail) + super()._configure(options, dont_fail) if self.binary: warnings.warn( "PythonItemExporter will drop support for binary export in the future", diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index dcaaeddfa..6bf9e5346 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -8,7 +8,7 @@ class Headers(CaselessDict): def __init__(self, seq=None, encoding='utf-8'): self.encoding = encoding - super(Headers, self).__init__(seq) + super().__init__(seq) def normkey(self, key): """Normalize key to bytes""" @@ -37,19 +37,19 @@ class Headers(CaselessDict): def __getitem__(self, key): try: - return super(Headers, self).__getitem__(key)[-1] + return super().__getitem__(key)[-1] except IndexError: return None def get(self, key, def_val=None): try: - return super(Headers, self).get(key, def_val)[-1] + return super().get(key, def_val)[-1] except IndexError: return None def getlist(self, key, def_val=None): try: - return super(Headers, self).__getitem__(key) + return super().__getitem__(key) except KeyError: if def_val is not None: return self.normvalue(def_val) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a260798ac..59af81321 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -24,7 +24,7 @@ class FormRequest(Request): if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' - super(FormRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if formdata: items = formdata.items() if isinstance(formdata, dict) else formdata diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index f08b25280..eae3f9f6b 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -32,7 +32,7 @@ class JsonRequest(Request): if 'method' not in kwargs: kwargs['method'] = 'POST' - super(JsonRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') @@ -47,7 +47,7 @@ class JsonRequest(Request): elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) - return super(JsonRequest, self).replace(*args, **kwargs) + return super().replace(*args, **kwargs) def _dumps(self, data): """Convert to JSON """ diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 811d3ad6b..c70912e49 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -31,5 +31,5 @@ class XmlRpcRequest(Request): if encoding is not None: kwargs['encoding'] = encoding - super(XmlRpcRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'text/xml') diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 0f300c8da..a7bb34d48 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -35,13 +35,13 @@ class TextResponse(Response): self._cached_benc = None self._cached_ubody = None self._cached_selector = None - super(TextResponse, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def _set_url(self, url): if isinstance(url, str): self._url = to_unicode(url, self.encoding) else: - super(TextResponse, self)._set_url(url) + super()._set_url(url) def _set_body(self, body): self._body = b'' # used by encoding detection @@ -51,7 +51,7 @@ class TextResponse(Response): type(self).__name__) self._body = body.encode(self._encoding) else: - super(TextResponse, self)._set_body(body) + super()._set_body(body) def replace(self, *args, **kwargs): kwargs.setdefault('encoding', self.encoding) @@ -166,7 +166,7 @@ class TextResponse(Response): elif isinstance(url, parsel.SelectorList): raise ValueError("SelectorList is not supported") encoding = self.encoding if encoding is None else encoding - return super(TextResponse, self).follow( + return super().follow( url=url, callback=callback, method=method, @@ -226,7 +226,7 @@ class TextResponse(Response): for sel in selectors: with suppress(_InvalidSelector): urls.append(_url_from_selector(sel)) - return super(TextResponse, self).follow_all( + return super().follow_all( urls=urls, callback=callback, method=method, diff --git a/scrapy/item.py b/scrapy/item.py index 4ab83d1a0..c262a153c 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta): if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)): warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', ScrapyDeprecationWarning, stacklevel=2) - return super(BaseItem, cls).__new__(cls, *args, **kwargs) + return super().__new__(cls, *args, **kwargs) class Field(dict): @@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta): def __new__(mcs, class_name, bases, attrs): classcell = attrs.pop('__classcell__', None) new_bases = tuple(base._class for base in bases if hasattr(base, '_class')) - _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs) + _class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs) fields = getattr(_class, 'fields', {}) new_attrs = {} @@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta): new_attrs['_class'] = _class if classcell is not None: new_attrs['__classcell__'] = classcell - return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs) + return super().__new__(mcs, class_name, bases, new_attrs) class DictItem(MutableMapping, BaseItem): @@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem): if issubclass(cls, DictItem) and not issubclass(cls, Item): warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead', ScrapyDeprecationWarning, stacklevel=2) - return super(DictItem, cls).__new__(cls, *args, **kwargs) + return super().__new__(cls, *args, **kwargs) def __init__(self, *args, **kwargs): self._values = {} @@ -109,7 +109,7 @@ class DictItem(MutableMapping, BaseItem): def __setattr__(self, name, value): if not name.startswith('_'): raise AttributeError("Use item[%r] = %r to set field value" % (name, value)) - super(DictItem, self).__setattr__(name, value) + super().__setattr__(name, value) def __len__(self): return len(self._values) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 984a5c4e1..08a6ca1e8 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -65,7 +65,7 @@ class FilteringLinkExtractor: warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' 'please use scrapy.linkextractors.LinkExtractor instead', ScrapyDeprecationWarning, stacklevel=2) - return super(FilteringLinkExtractor, cls).__new__(cls) + return super().__new__(cls) def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 8b9f961ee..e941c4321 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor): strip=strip, canonicalized=canonicalize ) - super(LxmlLinkExtractor, self).__init__( + super().__init__( link_extractor=lx, allow=allow, deny=deny, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 487382a38..6bc5d46eb 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -376,7 +376,7 @@ class FilesPipeline(MediaPipeline): resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD ) - super(FilesPipeline, self).__init__(download_func=download_func, settings=settings) + super().__init__(download_func=download_func, settings=settings) @classmethod def from_settings(cls, settings): diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 46f2bfb58..e2dd70215 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline): DEFAULT_IMAGES_RESULT_FIELD = 'images' def __init__(self, store_uri, download_func=None, settings=None): - super(ImagesPipeline, self).__init__(store_uri, settings=settings, - download_func=download_func) + super().__init__(store_uri, settings=settings, download_func=download_func) if isinstance(settings, dict) or settings is None: settings = Settings(settings) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f69894b1e..f191deac6 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver): """ def __init__(self, reactor, cache_size, timeout): - super(CachingThreadedResolver, self).__init__(reactor) + super().__init__(reactor) dnscache.limit = cache_size self.timeout = timeout @@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver): # so the input argument above is simply overridden # to enforce Scrapy's DNS_TIMEOUT setting's value timeout = (self.timeout,) - d = super(CachingThreadedResolver, self).getHostByName(name, timeout) + d = super().getHostByName(name, timeout) if dnscache.limit: d.addCallback(self._cache_result, name) return d @@ -80,16 +80,16 @@ class CachingHostnameResolver: class CachingResolutionReceiver(resolutionReceiver): def resolutionBegan(self, resolution): - super(CachingResolutionReceiver, self).resolutionBegan(resolution) + super().resolutionBegan(resolution) self.resolution = resolution self.resolved = False def addressResolved(self, address): - super(CachingResolutionReceiver, self).addressResolved(address) + super().addressResolved(address) self.resolved = True def resolutionComplete(self): - super(CachingResolutionReceiver, self).resolutionComplete() + super().resolutionComplete() if self.resolved: dnscache[hostName] = self.resolution diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 85a9bb526..f12c61081 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref): kwargs.setdefault('base_url', response.url) self.response = response - super(Selector, self).__init__(text=text, type=st, root=root, **kwargs) + super().__init__(text=text, type=st, root=root, **kwargs) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index ff8317cd1..b8ae32d7c 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -439,7 +439,7 @@ class Settings(BaseSettings): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user - super(Settings, self).__init__() + super().__init__() self.setmodule(default_settings, 'default') # Promote default dictionaries to BaseSettings instances for per-key # priorities diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 375042340..db9d0f2ae 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -15,7 +15,7 @@ class HttpError(IgnoreRequest): def __init__(self, response, *args, **kwargs): self.response = response - super(HttpError, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class HttpErrorMiddleware: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index cb7260892..c9fbce08d 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -75,7 +75,7 @@ class CrawlSpider(Spider): rules = () def __init__(self, *a, **kw): - super(CrawlSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self._compile_rules() def _parse(self, response, **kwargs): @@ -145,6 +145,6 @@ class CrawlSpider(Spider): @classmethod def from_crawler(cls, crawler, *args, **kwargs): - spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs) + spider = super().from_crawler(crawler, *args, **kwargs) spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True) return spider diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index fd41133ea..fe8c94e78 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -6,7 +6,7 @@ class InitSpider(Spider): """Base Spider with initialization facilities""" def start_requests(self): - self._postinit_reqs = super(InitSpider, self).start_requests() + self._postinit_reqs = super().start_requests() return iterate_spider_output(self.init_request()) def initialized(self, response=None): diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index c5360bfa7..1f72e76b7 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -18,7 +18,7 @@ class SitemapSpider(Spider): sitemap_alternate_links = False def __init__(self, *a, **kw): - super(SitemapSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: if isinstance(c, str): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index c7ad4d53d..77ffda6f7 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -20,7 +20,7 @@ def _with_mkdir(queue_class): if not os.path.exists(dirname): os.makedirs(dirname, exist_ok=True) - super(DirectoriesCreated, self).__init__(path, *args, **kwargs) + super().__init__(path, *args, **kwargs) return DirectoriesCreated @@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize): def push(self, obj): s = serialize(obj) - super(SerializableQueue, self).push(s) + super().push(s) def pop(self): - s = super(SerializableQueue, self).pop() + s = super().pop() if s: return deserialize(s) @@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class): def __init__(self, crawler, key): self.spider = crawler.spider - super(ScrapyRequestQueue, self).__init__(key) + super().__init__(key) @classmethod def from_crawler(cls, crawler, key, *args, **kwargs): @@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class): def push(self, request): request = request_to_dict(request, self.spider) - return super(ScrapyRequestQueue, self).push(request) + return super().push(request) def pop(self): - request = super(ScrapyRequestQueue, self).pop() + request = super().pop() if not request: return None diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 579c60180..ba7d1a6bf 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -54,7 +54,7 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): def __init__(self, crawler): - super(MemoryStatsCollector, self).__init__(crawler) + super().__init__(crawler) self.spider_stats = {} def _persist_stats(self, stats, spider): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 2a92d0588..e31284a7f 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -15,7 +15,7 @@ class CaselessDict(dict): __slots__ = () def __init__(self, seq=None): - super(CaselessDict, self).__init__() + super().__init__() if seq: self.update(seq) @@ -53,7 +53,7 @@ class CaselessDict(dict): def update(self, seq): seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) - super(CaselessDict, self).update(iseq) + super().update(iseq) @classmethod def fromkeys(cls, keys, value=None): @@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict): """ def __init__(self, limit=None): - super(LocalCache, self).__init__() + super().__init__() self.limit = limit def __setitem__(self, key, value): if self.limit: while len(self) >= self.limit: self.popitem(last=False) - super(LocalCache, self).__setitem__(key, value) + super().__setitem__(key, value) class LocalWeakReferencedCache(weakref.WeakKeyDictionary): @@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): """ def __init__(self, limit=None): - super(LocalWeakReferencedCache, self).__init__() + super().__init__() self.data = LocalCache(limit=limit) def __setitem__(self, key, value): try: - super(LocalWeakReferencedCache, self).__setitem__(key, value) + super().__setitem__(key, value) except TypeError: pass # key is not weak-referenceable, skip caching def __getitem__(self, key): try: - return super(LocalWeakReferencedCache, self).__getitem__(key) + return super().__getitem__(key) except (TypeError, KeyError): return None # key is either not weak-referenceable or not cached diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3dbea5fee..3c8e3c8b5 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -57,7 +57,7 @@ def create_deprecated_class( warned_on_subclass = False def __new__(metacls, name, bases, clsdict_): - cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_) + cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls @@ -73,7 +73,7 @@ def create_deprecated_class( if warn_once: msg += ' (warning only on first subclass, there may be others)' warnings.warn(msg, warn_category, stacklevel=2) - super(DeprecatedClass, cls).__init__(name, bases, clsdict_) + super().__init__(name, bases, clsdict_) # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks @@ -88,7 +88,7 @@ def create_deprecated_class( # is the deprecated class itself - subclasses of the # deprecated class should not use custom `__subclasscheck__` # method. - return super(DeprecatedClass, cls).__subclasscheck__(sub) + return super().__subclasscheck__(sub) if not inspect.isclass(sub): raise TypeError("issubclass() arg 1 must be a class") @@ -102,7 +102,7 @@ def create_deprecated_class( msg = instance_warn_message.format(cls=_clspath(cls, old_class_path), new=_clspath(new_class, new_class_path)) warnings.warn(msg, warn_category, stacklevel=2) - return super(DeprecatedClass, cls).__call__(*args, **kwargs) + return super().__call__(*args, **kwargs) deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {}) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 51d276097..1d6a2c39d 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -176,7 +176,7 @@ class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" def __init__(self, crawler, *args, **kwargs): - super(LogCounterHandler, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.crawler = crawler def emit(self, record): diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index dc9604578..cc3263602 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -33,7 +33,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): elif isinstance(o, Response): return "<%s %s %s>" % (type(o).__name__, o.status, o.url) else: - return super(ScrapyJSONEncoder, self).default(o) + return super().default(o) class ScrapyJSONDecoder(json.JSONDecoder): diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 66930ad2c..397e54703 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -7,12 +7,12 @@ class SiteTest: def setUp(self): from twisted.internet import reactor - super(SiteTest, self).setUp() + super().setUp() self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") self.baseurl = "http://localhost:%d/" % self.site.getHost().port def tearDown(self): - super(SiteTest, self).tearDown() + super().tearDown() self.site.stopListening() def url(self, path): diff --git a/tests/spiders.py b/tests/spiders.py index 3eb681819..63bd726fb 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -19,7 +19,7 @@ from scrapy.utils.test import get_from_asyncio_queue class MockServerSpider(Spider): def __init__(self, mockserver=None, *args, **kwargs): - super(MockServerSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.mockserver = mockserver @@ -28,7 +28,7 @@ class MetaSpider(MockServerSpider): name = 'meta' def __init__(self, *args, **kwargs): - super(MetaSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.meta = {} def closed(self, reason): @@ -41,7 +41,7 @@ class FollowAllSpider(MetaSpider): link_extractor = LinkExtractor() def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs): - super(FollowAllSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} @@ -60,7 +60,7 @@ class DelaySpider(MetaSpider): name = 'delay' def __init__(self, n=1, b=0, *args, **kwargs): - super(DelaySpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.n = n self.b = b self.t1 = self.t2 = self.t2_err = 0 @@ -82,7 +82,7 @@ class SimpleSpider(MetaSpider): name = 'simple' def __init__(self, url="http://localhost:8998", *args, **kwargs): - super(SimpleSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.start_urls = [url] def parse(self, response): @@ -153,7 +153,7 @@ class ItemSpider(FollowAllSpider): name = 'item' def parse(self, response): - for request in super(ItemSpider, self).parse(response): + for request in super().parse(response): yield request yield Item() yield {} @@ -172,7 +172,7 @@ class ErrorSpider(FollowAllSpider): raise self.exception_cls('Expected exception') def parse(self, response): - for request in super(ErrorSpider, self).parse(response): + for request in super().parse(response): yield request self.raise_exception() @@ -183,7 +183,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): fail_yielding = False def __init__(self, *a, **kw): - super(BrokenStartRequestsSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self.seedsseen = [] def start_requests(self): @@ -201,7 +201,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): def parse(self, response): self.seedsseen.append(response.meta.get('seed')) - for req in super(BrokenStartRequestsSpider, self).parse(response): + for req in super().parse(response): yield req @@ -243,7 +243,7 @@ class DuplicateStartRequestsSpider(MockServerSpider): yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): - super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.visited = 0 def parse(self, response): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 5754a5478..e115f420f 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -17,7 +17,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' def setUp(self): - super(ParseCommandTest, self).setUp() + super().setUp() self.spider_name = 'parse_spider' fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) with open(fname, 'w') as f: diff --git a/tests/test_commands.py b/tests/test_commands.py index 42091ab00..8938156fc 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -151,7 +151,7 @@ def get_permissions_dict(path, renamings=None, ignore=None): class StartprojectTemplatesTest(ProjectTest): def setUp(self): - super(StartprojectTemplatesTest, self).setUp() + super().setUp() self.tmpl = join(self.temp_path, 'templates') self.tmpl_proj = join(self.tmpl, 'project') @@ -315,7 +315,7 @@ class StartprojectTemplatesTest(ProjectTest): class CommandTest(ProjectTest): def setUp(self): - super(CommandTest, self).setUp() + super().setUp() self.call('startproject', self.project_name) self.cwd = join(self.temp_path, self.project_name) self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 99120b128..2e7e3ccc4 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -378,7 +378,7 @@ class ContractsManagerTest(unittest.TestCase): name = 'test_same_url' def __init__(self, *args, **kwargs): - super(TestSameUrlSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.visited = 0 def start_requests(s): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 57d4cdd6b..13063d106 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -530,7 +530,7 @@ class Https11InvalidDNSId(Https11TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" def setUp(self): - super(Https11InvalidDNSId, self).setUp() + super().setUp() self.host = '127.0.0.1' @@ -549,7 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase): 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' ) - super(Https11InvalidDNSPattern, self).setUp() + super().setUp() class Https11CustomCiphers(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9b77c97a8..299fb0eb8 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -134,7 +134,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): def _get_settings(self, **new_settings): new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module) - return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings) + return super()._get_settings(**new_settings) def test_custom_dbm_module_loaded(self): # make sure our dbm module has been loaded @@ -151,7 +151,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest): def _get_settings(self, **new_settings): new_settings.setdefault('HTTPCACHE_GZIP', True) - return super(FilesystemStorageTest, self)._get_settings(**new_settings) + return super()._get_settings(**new_settings) class DummyPolicyTest(_BaseTest): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index f9936baba..858138f81 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -189,7 +189,7 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): skip = "Rerp parser is not installed" def setUp(self): - super(RobotsTxtMiddlewareWithRerpTest, self).setUp() + super().setUp() self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser') @@ -198,5 +198,5 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest): skip = "Reppy parser is not installed" def setUp(self): - super(RobotsTxtMiddlewareWithReppyTest, self).setUp() + super().setUp() self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser') diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 660c99ce1..6c25a0064 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -593,7 +593,7 @@ class CustomExporterItemTest(unittest.TestCase): if name == 'age': return str(int(value) + 1) else: - return super(CustomItemExporter, self).serialize_field(field, name, value) + return super().serialize_field(field, name, value) i = self.item_class(name='John', age='22') a = ItemAdapter(i) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index f5cf4e798..0a303dbe2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1265,7 +1265,7 @@ class JsonRequestTest(RequestTest): def setUp(self): warnings.simplefilter("always") - super(JsonRequestTest, self).setUp() + super().setUp() def test_data(self): r1 = self.request_class(url="http://www.example.com/") @@ -1419,7 +1419,7 @@ class JsonRequestTest(RequestTest): def tearDown(self): warnings.resetwarnings() - super(JsonRequestTest, self).tearDown() + super().tearDown() if __name__ == "__main__": diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 56d017de6..f831ef5dc 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -305,7 +305,7 @@ class TextResponseTest(BaseResponseTest): response_class = TextResponse def test_replace(self): - super(TextResponseTest, self).test_replace() + super().test_replace() r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852") r2 = r1.replace(url="http://www.example.com/other") r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") diff --git a/tests/test_item.py b/tests/test_item.py index 0ce78f8c0..66fa761f0 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -312,7 +312,7 @@ class ItemMetaClassCellRegression(unittest.TestCase): # requirement. When not done properly raises an error: # TypeError: __class__ set to # defining 'MyItem' as - super(MyItem, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class DictItemTest(unittest.TestCase): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index a0bafa5e5..6f133d77a 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -516,7 +516,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ]) def test_restrict_xpaths_with_html_entities(self): - super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() + super().test_restrict_xpaths_with_html_entities() def test_filteringlinkextractor_deprecation_warning(self): """Make sure the FilteringLinkExtractor deprecation warning is not diff --git a/tests/test_loader.py b/tests/test_loader.py index 2ed6f365f..b0bc82f4e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -250,7 +250,7 @@ class TestOutputProcessorItem(unittest.TestCase): temp = Field() def __init__(self, *args, **kwargs): - super(TempItem, self).__init__(self, *args, **kwargs) + super().__init__(self, *args, **kwargs) self.setdefault('temp', 0.3) class TempLoader(ItemLoader): diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index eb14de14f..624dd9ab8 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -579,7 +579,7 @@ class TestOutputProcessorDict(unittest.TestCase): class TempDict(dict): def __init__(self, *args, **kwargs): - super(TempDict, self).__init__(self, *args, **kwargs) + super().__init__(self, *args, **kwargs) self.setdefault('temp', 0.3) class TempLoader(ItemLoader): diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index b771e7d79..41ff3651d 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -118,7 +118,7 @@ class LogFormatterTestCase(unittest.TestCase): class LogFormatterSubclass(LogFormatter): def crawled(self, request, response, spider): - kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider) + kwargs = super().crawled(request, response, spider) CRAWLEDMSG = ( "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" ) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3364d2258..b2b75ef20 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -53,7 +53,7 @@ class TestMiddlewareManager(MiddlewareManager): return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] def _add_middleware(self, mw): - super(TestMiddlewareManager, self)._add_middleware(mw) + super()._add_middleware(mw) if hasattr(mw, 'process'): self.methods['process'].append(mw.process) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 19ff00350..4f130c0c9 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -162,18 +162,18 @@ class BaseMediaPipelineTestCase(unittest.TestCase): class MockedMediaPipeline(MediaPipeline): def __init__(self, *args, **kwargs): - super(MockedMediaPipeline, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._mockcalled = [] def download(self, request, info): self._mockcalled.append('download') - return super(MockedMediaPipeline, self).download(request, info) + return super().download(request, info) def media_to_download(self, request, info): self._mockcalled.append('media_to_download') if 'result' in request.meta: return request.meta.get('result') - return super(MockedMediaPipeline, self).media_to_download(request, info) + return super().media_to_download(request, info) def get_media_requests(self, item, info): self._mockcalled.append('get_media_requests') @@ -181,15 +181,15 @@ class MockedMediaPipeline(MediaPipeline): def media_downloaded(self, response, request, info): self._mockcalled.append('media_downloaded') - return super(MockedMediaPipeline, self).media_downloaded(response, request, info) + return super().media_downloaded(response, request, info) def media_failed(self, failure, request, info): self._mockcalled.append('media_failed') - return super(MockedMediaPipeline, self).media_failed(failure, request, info) + return super().media_failed(failure, request, info) def item_completed(self, results, item, info): self._mockcalled.append('item_completed') - item = super(MockedMediaPipeline, self).item_completed(results, item, info) + item = super().item_completed(results, item, info) item['results'] = results return item diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 5cfef8e7d..373b2e49c 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -10,7 +10,7 @@ class SignalCatcherSpider(Spider): name = 'signal_catcher' def __init__(self, crawler, url, *args, **kwargs): - super(SignalCatcherSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) crawler.signals.connect(self.on_request_left, signal=request_left_downloader) self.caught_times = 0 diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 9d8c201dd..4b15d0fab 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -117,7 +117,7 @@ class BaseRobotParserTest: class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import PythonRobotParser - super(PythonRobotParserTest, self)._setUp(PythonRobotParser) + super()._setUp(PythonRobotParser) def test_length_based_precedence(self): raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.") @@ -132,7 +132,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import ReppyRobotParser - super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser) + super()._setUp(ReppyRobotParser) def test_order_based_precedence(self): raise unittest.SkipTest("Reppy does not support order based directives precedence.") @@ -144,7 +144,7 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import RerpRobotParser - super(RerpRobotParserTest, self)._setUp(RerpRobotParser) + super()._setUp(RerpRobotParser) def test_length_based_precedence(self): raise unittest.SkipTest("Rerp does not support length based directives precedence.") @@ -156,7 +156,7 @@ class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import ProtegoRobotParser - super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser) + super()._setUp(ProtegoRobotParser) def test_order_based_precedence(self): raise unittest.SkipTest("Protego does not support order based directives precedence.") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 2b6cb0902..512a7460e 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -53,7 +53,7 @@ class MockCrawler(Crawler): JOBDIR=jobdir, DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', ) - super(MockCrawler, self).__init__(Spider, settings) + super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) @@ -296,7 +296,7 @@ class StartUrlsSpider(Spider): def __init__(self, start_urls): self.start_urls = start_urls - super(StartUrlsSpider, self).__init__(name='StartUrlsSpider') + super().__init__(name='StartUrlsSpider') def parse(self, response): pass diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e032b247c..e449cd706 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -19,7 +19,7 @@ class _HttpErrorSpider(MockServerSpider): bypass_status_codes = set() def __init__(self, *args, **kwargs): - super(_HttpErrorSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.start_urls = [ self.mockserver.url("/status?n=200"), self.mockserver.url("/status?n=404"), From 9d84289109b2368d5929d8b60ce583529c19fe4c Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 5 Aug 2020 09:11:59 +0530 Subject: [PATCH 219/568] deprecated weakkeycache by specifying in __init__ --- scrapy/utils/python.py | 4 +++- scrapy/utils/tester.py | 3 +++ tests/test_utils_python.py | 22 +++++++++++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 scrapy/utils/tester.py diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index c8f921ff3..4756b07b6 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -7,10 +7,12 @@ import inspect import re import sys import weakref +import warnings from functools import partial, wraps from itertools import chain from scrapy.utils.decorators import deprecated +from scrapy.exceptions import ScrapyDeprecationWarning def flatten(x): @@ -275,10 +277,10 @@ def equal_attributes(obj1, obj2, attributes): return True -@deprecated class WeakKeyCache: def __init__(self, default_factory): + warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2) self.default_factory = default_factory self._weakdict = weakref.WeakKeyDictionary() diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py new file mode 100644 index 000000000..691e9bc1a --- /dev/null +++ b/scrapy/utils/tester.py @@ -0,0 +1,3 @@ +from scrapy.utils.decorators import deprecated + + diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 5a53d89e4..ebce3c079 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,15 +1,18 @@ import functools +import gc import operator import platform import unittest +from itertools import count from sys import version_info from warnings import catch_warnings from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - get_func_args, to_bytes, to_unicode, + WeakKeyCache, get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) + __doctests__ = ['scrapy.utils.python'] @@ -152,6 +155,23 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) + def test_weakkeycache(self): + class _Weakme: + pass + + _values = count() + wk = WeakKeyCache(lambda k: next(_values)) + k = _Weakme() + v = wk[k] + self.assertEqual(v, wk[k]) + self.assertNotEqual(v, wk[_Weakme()]) + self.assertEqual(v, wk[k]) + del k + for _ in range(100): + if wk._weakdict: + gc.collect() + self.assertFalse(len(wk._weakdict)) + def test_get_func_args(self): def f1(a, b, c): pass From b35d1f2b2c430f4d12cb8f9d408dfa0c0051746d Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 5 Aug 2020 09:14:04 +0530 Subject: [PATCH 220/568] deleted tester.py --- scrapy/utils/tester.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 scrapy/utils/tester.py diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py deleted file mode 100644 index 691e9bc1a..000000000 --- a/scrapy/utils/tester.py +++ /dev/null @@ -1,3 +0,0 @@ -from scrapy.utils.decorators import deprecated - - From 983b7ddf2e39c480efc6d104054f92f570714ac8 Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 5 Aug 2020 16:13:52 +0530 Subject: [PATCH 221/568] aesthetic fixes --- scrapy/utils/python.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 4756b07b6..59f1b8371 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -6,13 +6,13 @@ import gc import inspect import re import sys -import weakref import warnings +import weakref from functools import partial, wraps from itertools import chain -from scrapy.utils.decorators import deprecated from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.decorators import deprecated def flatten(x): @@ -280,7 +280,7 @@ def equal_attributes(obj1, obj2, attributes): class WeakKeyCache: def __init__(self, default_factory): - warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2) + warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2) self.default_factory = default_factory self._weakdict = weakref.WeakKeyDictionary() From 4dc09f09aa9698b02f2cbf2e3001202388eba043 Mon Sep 17 00:00:00 2001 From: linchiwei123 <40888469+linchiwei123@users.noreply.github.com> Date: Wed, 5 Aug 2020 22:23:19 +0800 Subject: [PATCH 222/568] Update setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 58090f7a2..d0880051f 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ install_requires = [ 'cryptography>=2.0', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', - 'lxml>=3.5.0', 'parsel>=1.5.0', 'PyDispatcher>=2.0.5', 'pyOpenSSL>=16.2.0', From 13181ba7882f2aef3ea103a9c2391fd8f87fbb44 Mon Sep 17 00:00:00 2001 From: Jose Galdos Date: Thu, 23 Jul 2020 18:45:45 -0500 Subject: [PATCH 223/568] Improve http status all on http error middleware --- scrapy/spidermiddlewares/httperror.py | 2 +- tests/test_spidermiddleware_httperror.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 375042340..bf908d2f7 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -32,7 +32,7 @@ class HttpErrorMiddleware: if 200 <= response.status < 300: # common case return meta = response.meta - if 'handle_httpstatus_all' in meta: + if meta.get('handle_httpstatus_all', False): return if 'handle_httpstatus_list' in meta: allowed_statuses = meta['handle_httpstatus_list'] diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e032b247c..f3e5478c4 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -139,6 +139,19 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + def test_httperror_allow_all_false(self): + crawler = get_crawler(_HttpErrorSpider) + mw = HttpErrorMiddleware.from_crawler(crawler) + request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False}) + request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True}) + res404 = self.res404.copy() + res404.request = request_httpstatus_false + res402 = self.res402.copy() + res402.request = request_httpstatus_true + + self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider) + self.assertIsNone(mw.process_spider_input(res402, self.spider)) + class TestHttpErrorMiddlewareIntegrational(TrialTestCase): def setUp(self): From e0c3019d90f187482cd84b946b83c496411ec34b Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 9 Aug 2020 16:19:35 +0530 Subject: [PATCH 224/568] fix: ScrapyProxyH2Agent - add required test cases BREAKING CHANGES Presently the tests (in test_downloader_handlers.py) 1. test_download_without_proxy 2. test_download_with_proxy_https_timeout collide with each other when run together. However, if both of the tests are ran individually then both pass. --- scrapy/core/downloader/handlers/http2.py | 14 ++++++--- scrapy/core/http2/agent.py | 15 ++++++---- scrapy/core/http2/stream.py | 22 +++++++++++++-- tests/test_downloader_handlers.py | 36 ++++++++++++++++++++---- 4 files changed, 70 insertions(+), 17 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 411e06a78..d6e5cd1c1 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -6,7 +6,7 @@ from urllib.parse import urldefrag from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.web.client import URI +from twisted.web.client import URI, BrowserLikePolicyForHTTPS from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse @@ -45,19 +45,24 @@ class ScrapyProxyH2Agent(H2Agent): def __init__( self, reactor: ReactorBase, proxy_uri: URI, pool: H2ConnectionPool, + context_factory=BrowserLikePolicyForHTTPS(), connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None ) -> None: super(ScrapyProxyH2Agent, self).__init__( reactor=reactor, pool=pool, + context_factory=context_factory, connect_timeout=connect_timeout, bind_address=bind_address ) self._proxy_uri = proxy_uri - @staticmethod - def get_key(uri: URI) -> Tuple: - return "http-proxy", uri.host, uri.port + def get_endpoint(self, uri: URI): + return self.endpoint_factory.endpointForURI(self._proxy_uri) + + def get_key(self, uri: URI) -> Tuple: + """We use the proxy uri instead of uri obtained from request url""" + return "http-proxy", self._proxy_uri.host, self._proxy_uri.port class ScrapyH2Agent: @@ -100,6 +105,7 @@ class ScrapyH2Agent: else: return self._ProxyAgent( reactor=reactor, + context_factory=self._context_factory, proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')), connect_timeout=timeout, bind_address=bind_address, diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index aa51508a5..f4ac29bc6 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -118,22 +118,25 @@ class H2Agent: self._reactor = reactor self._pool = pool self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2']) - self._endpoint_factory = _StandardEndpointFactory( + self.endpoint_factory = _StandardEndpointFactory( self._reactor, self._context_factory, connect_timeout, bind_address ) - def _get_endpoint(self, uri: URI): - return self._endpoint_factory.endpointForURI(uri) + def get_endpoint(self, uri: URI): + return self.endpoint_factory.endpointForURI(uri) - @staticmethod - def get_key(uri: URI) -> Tuple: + def get_key(self, uri: URI) -> Tuple: + """ + Arguments: + uri - URI obtained directly from request URL + """ return uri.scheme, uri.host, uri.port def request(self, request: Request, spider: Spider) -> Deferred: uri = URI.fromBytes(bytes(request.url, encoding='utf-8')) try: - endpoint = self._get_endpoint(uri) + endpoint = self.get_endpoint(uri) except SchemeNotSupported: return defer.fail(Failure()) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index acdd46320..40ea07b63 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -185,14 +185,32 @@ class Stream: if url.query: path += '?' + url.query + # This pseudo-header field MUST NOT be empty for "http" or "https" + # URIs; "http" or "https" URIs that do not contain a path component + # MUST include a value of '/'. The exception to this rule is an + # OPTIONS request for an "http" or "https" URI that does not include + # a path component; these MUST include a ":path" pseudo-header field + # with a value of '*' (refer RFC 7540 - Section 8.1.2.3) + if not path: + if self._request.method == 'OPTIONS': + path = path or '*' + else: + path = path or '/' + # Make sure pseudo-headers comes before all the other headers headers = [ (':method', self._request.method), (':authority', url.netloc), - (':scheme', self._protocol.metadata['uri'].scheme), - (':path', path), ] + # The ":scheme" and ":path" pseudo-header fields MUST + # be omitted for CONNECT method (refer RFC 7540 - Section 8.3) + if self._request.method != 'CONNECT': + headers += [ + (':scheme', self._protocol.metadata['uri'].scheme), + (':path', path), + ] + for name, value in self._request.headers.items(): headers.append((str(name, 'utf-8'), str(value[0], 'utf-8'))) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 4dd32b6f5..486614121 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -786,7 +786,10 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'http://example.com') + self.assertTrue( + response.body == b'http://example.com' # HTTP/1.x + or response.body == b'/' # HTTP/2 + ) http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -796,10 +799,13 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'https://example.com') + self.assertTrue( + response.body == b'http://example.com' # HTTP/1.x + or response.body == b'/' # HTTP/2 + ) http_proxy = '%s?noconnect' % self.getURL('') - request = Request('https://example.com', meta={'proxy': http_proxy}) + request = Request('http://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex(ScrapyDeprecationWarning, r'Using HTTPS proxies in the noconnect mode is deprecated'): return self.download_request(request, Spider('foo')).addCallback(_test) @@ -836,10 +842,30 @@ class Http11ProxyTestCase(HttpProxyTestCase): self.assertIn(domain, timeout.osError) -# TODO: -class Http2ProxyTestCase(Http11ProxyTestCase): +class Https2ProxyTestCase(Http11ProxyTestCase): + # only used for HTTPS tests + keyfile = 'keys/localhost.key' + certfile = 'keys/localhost.crt' + + scheme = 'https' + host = u'127.0.0.1' + download_handler_cls = H2DownloadHandler + def setUp(self): + site = server.Site(UriResource(), timeout=None) + self.port = reactor.listenSSL( + 0, site, + ssl_context_factory(self.keyfile, self.certfile), + interface=self.host + ) + self.portno = self.port.getHost().port + self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) + self.download_request = self.download_handler.download_request + + def getURL(self, path): + return f"{self.scheme}://{self.host}:{self.portno}/{path}" + class HttpDownloadHandlerMock: From c67d6dea318d6f0915ae86d46ce367b6c1e8ee51 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 11 Aug 2020 04:39:41 +0530 Subject: [PATCH 225/568] fix: H2 docs, NotImplementedError for H2 Tunnel --- docs/topics/settings.rst | 15 ++++---- scrapy/core/downloader/contextfactory.py | 7 ++-- scrapy/core/downloader/handlers/http2.py | 44 +++++++++++------------- scrapy/core/http2/stream.py | 5 +-- tests/test_downloader_handlers.py | 33 ++++++++++++------ 5 files changed, 54 insertions(+), 50 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 670b44f3c..bb543433c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -620,8 +620,8 @@ handler (without replacement), place this in your ``settings.py``:: 'ftp': None, } -The default https handler uses HTTP/1.x, to use HTTP/2.0 update :setting:`DOWNLOAD_HANDLERS` -as:: +The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update +:setting:`DOWNLOAD_HANDLERS` as follows:: DOWNLOAD_HANDLERS = { 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', @@ -706,11 +706,12 @@ Optionally, this can be set per-request basis by using the .. warning:: - This is ignored when :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` - is set as ``https`` download handler in :setting:`DOWNLOAD_HANDLERS`. In - case of data loss error the connection may be corrupted affecting other streams, - hence all streams return with the ``ResponseFailed([InvalidBodyLengthError])`` - failure. + This setting is ignored by the + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` + download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss + error, the corresponding HTTP/2 connection may be corrupted, affecting other + requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])`` + failure is always raised for every request that was using that connection. .. setting:: DUPEFILTER_CLASS diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index c0463cfc7..8dcba15ff 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,14 +1,13 @@ -from OpenSSL import SSL import warnings +from OpenSSL import SSL from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer -from scrapy.core.downloader.tls import openssl_methods -from scrapy.utils.misc import create_instance, load_object -from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS +from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions +from scrapy.utils.misc import create_instance, load_object @implementer(IPolicyForHTTPS) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index d6e5cd1c1..650af9778 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -6,15 +6,15 @@ from urllib.parse import urldefrag from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.web.client import URI, BrowserLikePolicyForHTTPS +from twisted.web.client import BrowserLikePolicyForHTTPS, URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.python import to_bytes class H2DownloadHandler: @@ -88,29 +88,25 @@ class ScrapyH2Agent: if proxy: _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) scheme = _parse(request.url)[0] - proxy_host = str(proxy_host, 'utf-8') - omit_connect_timeout = b'noconnect' in proxy_params - if omit_connect_timeout: - warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Crawlera, it doesn't require this mode anymore, " - "so you should update scrapy-crawlera to 1.3.0+ " - "and remove '?noconnect' from the Crawlera URL.", - ScrapyDeprecationWarning) + proxy_host = proxy_host.decode() + omit_connect_tunnel = b'noconnect' in proxy_params + if omit_connect_tunnel: + warnings.warn("Using HTTPS proxies in the noconnect mode is not supported by the " + "downloader handler. If you use Crawlera, it doesn't require this " + "mode anymore, so you should update scrapy-crawlera to 1.3.0+ " + "and remove '?noconnect' from the Crawlera URL.") - if scheme == b'https' and not omit_connect_timeout: - proxy_auth = request.headers.get(b'Proxy-Authorization', None) - proxy_conf = (proxy_host, proxy_port, proxy_auth) - - # TODO: Return TunnelingAgent instance - else: - return self._ProxyAgent( - reactor=reactor, - context_factory=self._context_factory, - proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')), - connect_timeout=timeout, - bind_address=bind_address, - pool=self._pool - ) + if scheme == b'https' and not omit_connect_tunnel: + # ToDo + raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported') + return self._ProxyAgent( + reactor=reactor, + context_factory=self._context_factory, + proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')), + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool + ) return self._Agent( reactor=reactor, diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 40ea07b63..1e136fbd5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -192,10 +192,7 @@ class Stream: # a path component; these MUST include a ":path" pseudo-header field # with a value of '*' (refer RFC 7540 - Section 8.1.2.3) if not path: - if self._request.method == 'OPTIONS': - path = path or '*' - else: - path = path or '/' + path = '*' if self._request.method == 'OPTIONS' else '/' # Make sure pseudo-headers comes before all the other headers headers = [ diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 486614121..1cedf6b10 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -315,8 +315,8 @@ class HttpTestCase(unittest.TestCase): host = self.host + ':' + str(self.portno) def _test(response): - self.assertEqual(response.body, bytes(host, 'utf-8')) - self.assertEqual(request.headers.get('Host'), bytes(host, 'utf-8')) + self.assertEqual(response.body, to_bytes(host)) + self.assertEqual(request.headers.get('Host'), to_bytes(host)) request = Request(self.getURL('host'), headers={'Host': host}) return self.download_request(request, Spider('foo')).addCallback(_test) @@ -764,6 +764,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): download_handler_cls = HTTPDownloadHandler + expected_http_proxy_request_body = b'http://example.com' def setUp(self): site = server.Site(UriResource(), timeout=None) @@ -786,10 +787,7 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertTrue( - response.body == b'http://example.com' # HTTP/1.x - or response.body == b'/' # HTTP/2 - ) + self.assertEqual(response.body, self.expected_http_proxy_request_body) http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -799,13 +797,10 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertTrue( - response.body == b'http://example.com' # HTTP/1.x - or response.body == b'/' # HTTP/2 - ) + self.assertEqual(response.body, b'https://example.com') http_proxy = '%s?noconnect' % self.getURL('') - request = Request('http://example.com', meta={'proxy': http_proxy}) + request = Request('https://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex(ScrapyDeprecationWarning, r'Using HTTPS proxies in the noconnect mode is deprecated'): return self.download_request(request, Spider('foo')).addCallback(_test) @@ -851,6 +846,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): host = u'127.0.0.1' download_handler_cls = H2DownloadHandler + expected_http_proxy_request_body = b'/' def setUp(self): site = server.Site(UriResource(), timeout=None) @@ -866,6 +862,21 @@ class Https2ProxyTestCase(Http11ProxyTestCase): def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" + def test_download_with_proxy_https_noconnect(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'/') + + http_proxy = '%s?noconnect' % self.getURL('') + request = Request('https://example.com', meta={'proxy': http_proxy}) + with self.assertWarnsRegex( + Warning, + r'Using HTTPS proxies in the noconnect mode is not supported by the ' + r'downloader handler.' + ): + return self.download_request(request, Spider('foo')).addCallback(_test) + class HttpDownloadHandlerMock: From 90f85a2b9b0cfac4a7a56a1926e9694ef7e2d299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Aug 2020 10:20:30 +0200 Subject: [PATCH 226/568] Enable Travis CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b403ac54c..a3bd2e199 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ dist: xenial branches: only: - master + - http2 # Remove once merged into master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: From aefd43a6c6b8d2e32dfb9bd0c94529805ce5037e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Aug 2020 12:52:54 +0200 Subject: [PATCH 227/568] Upgrade minimum dependencies for Python 3.6 support --- tests/requirements-py3.txt | 3 +-- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 00c56084d..4fa58d11b 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,8 +1,7 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy; python_version >= '3.6' -mitmproxy<4.0.0; python_version < '3.6' +mitmproxy >=4, <5 # The latest version does not support some pinned dependencies # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines diff --git a/tox.ini b/tox.ini index 11882c03f..4f5531aea 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ deps = -rtests/requirements-py3.txt # Extras boto3>=1.13.0 - botocore>=1.3.23 + botocore>=1.4.87 Pillow>=3.4.2 passenv = S3_TEST_FILE_URI @@ -76,7 +76,7 @@ deps = zope.interface==4.1.3 -rtests/requirements-py3.txt # Extras - botocore==1.3.23 + botocore==1.4.87 google-cloud-storage==1.29.0 Pillow==3.4.2 From 1c4b4cc6b046e121be33a57031a0900ca1347e16 Mon Sep 17 00:00:00 2001 From: Ajay Mittur Date: Tue, 11 Aug 2020 17:42:44 +0530 Subject: [PATCH 228/568] Support defining file path based on item in media pipelines (#4686) --- docs/topics/media-pipeline.rst | 24 ++++--- scrapy/pipelines/files.py | 16 ++--- scrapy/pipelines/images.py | 14 ++-- scrapy/pipelines/media.py | 69 +++++++++++++++--- tests/test_pipeline_files.py | 13 ++++ tests/test_pipeline_media.py | 123 ++++++++++++++++++++++++++++++++- 6 files changed, 225 insertions(+), 34 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 1f995ce14..487e26b8e 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -412,15 +412,16 @@ See here the methods that you can override in your custom Files Pipeline: .. class:: FilesPipeline - .. method:: file_path(self, request, response=None, info=None) + .. method:: file_path(self, request, response=None, info=None, *, item=None) This method is called once per downloaded item. It returns the download path of the file originating from the specified :class:`response `. In addition to ``response``, this method receives the original - :class:`request ` and - :class:`info `. + :class:`request `, + :class:`info ` and + :class:`item ` You can override this method to customize the download path of each file. @@ -436,9 +437,12 @@ See here the methods that you can override in your custom Files Pipeline: class MyFilesPipeline(FilesPipeline): - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): return 'files/' + os.path.basename(urlparse(request.url).path) + Similarly, you can use the ``item`` to determine the file path based on some item + property. + By default the :meth:`file_path` method returns ``full/.``. @@ -544,15 +548,16 @@ See here the methods that you can override in your custom Images Pipeline: The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`, customizing the field names and adding custom behavior for images. - .. method:: file_path(self, request, response=None, info=None) + .. method:: file_path(self, request, response=None, info=None, *, item=None) This method is called once per downloaded item. It returns the download path of the file originating from the specified :class:`response `. In addition to ``response``, this method receives the original - :class:`request ` and - :class:`info `. + :class:`request `, + :class:`info ` and + :class:`item ` You can override this method to customize the download path of each file. @@ -568,9 +573,12 @@ See here the methods that you can override in your custom Images Pipeline: class MyImagesPipeline(ImagesPipeline): - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): return 'files/' + os.path.basename(urlparse(request.url).path) + Similarly, you can use the ``item`` to determine the file path based on some item + property. + By default the :meth:`file_path` method returns ``full/.``. diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 6bc5d46eb..5a2184681 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -409,7 +409,7 @@ class FilesPipeline(MediaPipeline): store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): def _onsuccess(result): if not result: return # returning None force download @@ -436,7 +436,7 @@ class FilesPipeline(MediaPipeline): checksum = result.get('checksum', None) return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'} - path = self.file_path(request, info=info) + path = self.file_path(request, info=info, item=item) dfd = defer.maybeDeferred(self.store.stat_file, path, info) dfd.addCallbacks(_onsuccess, lambda _: None) dfd.addErrback( @@ -460,7 +460,7 @@ class FilesPipeline(MediaPipeline): raise FileException - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): referer = referer_str(request) if response.status != 200: @@ -492,8 +492,8 @@ class FilesPipeline(MediaPipeline): self.inc_stats(info.spider, status) try: - path = self.file_path(request, response=response, info=info) - checksum = self.file_downloaded(response, request, info) + path = self.file_path(request, response=response, info=info, item=item) + checksum = self.file_downloaded(response, request, info, item=item) except FileException as exc: logger.warning( 'File (error): Error processing file from %(request)s ' @@ -522,8 +522,8 @@ class FilesPipeline(MediaPipeline): urls = ItemAdapter(item).get(self.files_urls_field, []) return [Request(u) for u in urls] - def file_downloaded(self, response, request, info): - path = self.file_path(request, response=response, info=info) + def file_downloaded(self, response, request, info, *, item=None): + path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) checksum = md5sum(buf) buf.seek(0) @@ -535,7 +535,7 @@ class FilesPipeline(MediaPipeline): ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() media_ext = os.path.splitext(request.url)[1] # Handles empty and wild extensions by trying to guess the diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e2dd70215..0a67a0b1d 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -103,12 +103,12 @@ class ImagesPipeline(FilesPipeline): store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) - def file_downloaded(self, response, request, info): - return self.image_downloaded(response, request, info) + def file_downloaded(self, response, request, info, *, item=None): + return self.image_downloaded(response, request, info, item=item) - def image_downloaded(self, response, request, info): + def image_downloaded(self, response, request, info, *, item=None): checksum = None - for path, image, buf in self.get_images(response, request, info): + for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) checksum = md5sum(buf) @@ -119,8 +119,8 @@ class ImagesPipeline(FilesPipeline): headers={'Content-Type': 'image/jpeg'}) return checksum - def get_images(self, response, request, info): - path = self.file_path(request, response=response, info=info) + def get_images(self, response, request, info, *, item=None): + path = self.file_path(request, response=response, info=info, item=item) orig_image = Image.open(BytesIO(response.body)) width, height = orig_image.size @@ -166,7 +166,7 @@ class ImagesPipeline(FilesPipeline): ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return 'full/%s.jpg' % (image_guid) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index aa65f4f0e..2439de9a5 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,12 +1,16 @@ import functools import logging from collections import defaultdict +from inspect import signature +from warnings import warn + from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter from scrapy.utils.log import failure_to_exc_info @@ -27,6 +31,7 @@ class MediaPipeline: def __init__(self, download_func=None, settings=None): self.download_func = download_func + self._expects_item = {} if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -38,6 +43,9 @@ class MediaPipeline: ) self._handle_statuses(self.allow_redirects) + # Check if deprecated methods are being used and make them compatible + self._make_compatible() + def _handle_statuses(self, allow_redirects): self.handle_httpstatus_list = None if allow_redirects: @@ -77,11 +85,11 @@ class MediaPipeline: def process_item(self, item, spider): info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) - dlist = [self._process_request(r, info) for r in requests] + dlist = [self._process_request(r, info, item) for r in requests] dfd = DeferredList(dlist, consumeErrors=1) return dfd.addCallback(self.item_completed, item, info) - def _process_request(self, request, info): + def _process_request(self, request, info, item): fp = request_fingerprint(request) cb = request.callback or (lambda _: _) eb = request.errback @@ -102,34 +110,73 @@ class MediaPipeline: # Download request checking media_to_download hook output first info.downloading.add(fp) - dfd = mustbe_deferred(self.media_to_download, request, info) - dfd.addCallback(self._check_media_to_download, request, info) + dfd = mustbe_deferred(self.media_to_download, request, info, item=item) + dfd.addCallback(self._check_media_to_download, request, info, item=item) dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) dfd.addErrback(lambda f: logger.error( f.value, exc_info=failure_to_exc_info(f), extra={'spider': info.spider}) ) return dfd.addBoth(lambda _: wad) # it must return wad at last + def _make_compatible(self): + """Make overridable methods of MediaPipeline and subclasses backwards compatible""" + methods = [ + "file_path", "media_to_download", "media_downloaded", + "file_downloaded", "image_downloaded", "get_images" + ] + + for method_name in methods: + method = getattr(self, method_name, None) + if callable(method): + setattr(self, method_name, self._compatible(method)) + + def _compatible(self, func): + """Wrapper for overridable methods to allow backwards compatibility""" + self._check_signature(func) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if self._expects_item[func.__name__]: + return func(*args, **kwargs) + + kwargs.pop('item', None) + return func(*args, **kwargs) + + return wrapper + + def _check_signature(self, func): + sig = signature(func) + self._expects_item[func.__name__] = True + + if 'item' not in sig.parameters: + old_params = str(sig)[1:-1] + new_params = old_params + ", *, item=None" + warn('%s(self, %s) is deprecated, ' + 'please use %s(self, %s)' + % (func.__name__, old_params, func.__name__, new_params), + ScrapyDeprecationWarning, stacklevel=2) + self._expects_item[func.__name__] = False + def _modify_media_request(self, request): if self.handle_httpstatus_list: request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list else: request.meta['handle_httpstatus_all'] = True - def _check_media_to_download(self, result, request, info): + def _check_media_to_download(self, result, request, info, item): if result is not None: return result if self.download_func: # this ugly code was left only to support tests. TODO: remove dfd = mustbe_deferred(self.download_func, request, info.spider) dfd.addCallbacks( - callback=self.media_downloaded, callbackArgs=(request, info), + callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) else: self._modify_media_request(request) dfd = self.crawler.engine.download(request, info.spider) dfd.addCallbacks( - callback=self.media_downloaded, callbackArgs=(request, info), + callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) return dfd @@ -171,7 +218,7 @@ class MediaPipeline: defer_result(result).chainDeferred(wad) # Overridable Interface - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): """Check request before starting download""" pass @@ -179,7 +226,7 @@ class MediaPipeline: """Returns the media requests to download""" pass - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): """Handler for success downloads""" return response @@ -199,3 +246,7 @@ class MediaPipeline: extra={'spider': info.spider} ) return item + + def file_path(self, request, response=None, info=None, *, item=None): + """Returns the path where downloaded media should be stored""" + pass diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index a023dfcc8..b19b4ff2a 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -161,6 +161,19 @@ class FilesPipelineTestCase(unittest.TestCase): for p in patchers: p.stop() + def test_file_path_from_item(self): + """ + Custom file path based on item data, overriding default implementation + """ + class CustomFilesPipeline(FilesPipeline): + def file_path(self, request, response=None, info=None, item=None): + return 'full/%s' % item.get('path') + + file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path + item = dict(path='path-to-store-file') + request = Request("http://example.com") + self.assertEqual(file_path(request, item=item), 'full/path-to-store-file') + class FilesPipelineTestCaseFieldsMixin: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 4f130c0c9..6afd47497 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -7,7 +7,9 @@ from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.request import request_fingerprint +from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.pipelines.files import FileException from scrapy.utils.log import failure_to_exc_info @@ -169,7 +171,7 @@ class MockedMediaPipeline(MediaPipeline): self._mockcalled.append('download') return super().download(request, info) - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): self._mockcalled.append('media_to_download') if 'result' in request.meta: return request.meta.get('result') @@ -179,7 +181,7 @@ class MockedMediaPipeline(MediaPipeline): self._mockcalled.append('get_media_requests') return item.get('requests') - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): self._mockcalled.append('media_downloaded') return super().media_downloaded(response, request, info) @@ -335,6 +337,123 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ['get_media_requests', 'media_to_download', 'item_completed']) +class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): + + def __init__(self, *args, **kwargs): + super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs) + self._mockcalled = [] + + def get_media_requests(self, item, info): + item_url = item['image_urls'][0] + return Request( + item_url, + meta={'response': Response(item_url, status=200, body=b'data')} + ) + + def inc_stats(self, *args, **kwargs): + return True + + def media_to_download(self, request, info): + self._mockcalled.append('media_to_download') + return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info) + + def media_downloaded(self, response, request, info): + self._mockcalled.append('media_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info) + + def file_downloaded(self, response, request, info): + self._mockcalled.append('file_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info) + + def file_path(self, request, response=None, info=None): + self._mockcalled.append('file_path') + return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info) + + def get_images(self, response, request, info): + self._mockcalled.append('get_images') + return [] + + def image_downloaded(self, response, request, info): + self._mockcalled.append('image_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info) + + +class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): + + def setUp(self): + self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func) + self.pipe.open_spider(None) + self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[]) + + def _assert_method_called_with_warnings(self, method, message, warnings): + self.assertIn(method, self.pipe._mockcalled) + warningShown = False + for warning in warnings: + if warning['message'] == message and warning['category'] == ScrapyDeprecationWarning: + warningShown = True + self.assertTrue(warningShown) + + @inlineCallbacks + def test_media_to_download_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'media_to_download(self, request, info) is deprecated, ' + 'please use media_to_download(self, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('media_to_download', message, warnings) + + @inlineCallbacks + def test_media_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'media_downloaded(self, response, request, info) is deprecated, ' + 'please use media_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('media_downloaded', message, warnings) + + @inlineCallbacks + def test_file_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'file_downloaded(self, response, request, info) is deprecated, ' + 'please use file_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('file_downloaded', message, warnings) + + @inlineCallbacks + def test_file_path_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'file_path(self, request, response=None, info=None) is deprecated, ' + 'please use file_path(self, request, response=None, info=None, *, item=None)' + ) + self._assert_method_called_with_warnings('file_path', message, warnings) + + @inlineCallbacks + def test_get_images_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'get_images(self, response, request, info) is deprecated, ' + 'please use get_images(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('get_images', message, warnings) + + @inlineCallbacks + def test_image_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'image_downloaded(self, response, request, info) is deprecated, ' + 'please use image_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('image_downloaded', message, warnings) + + class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): def _assert_request_no3xx(self, pipeline_class, settings): From 394631fc0a731a0b8b0108edc1711c0c87e2ca15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 12:08:09 +0200 Subject: [PATCH 229/568] Restore 3.5 support for mitmproxy-based tests --- tests/requirements-py3.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 4fa58d11b..b67c08403 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,7 +1,8 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy >=4, <5 # The latest version does not support some pinned dependencies +mitmproxy >=4, <5; python_version < '3.6' # The latest version does not support some pinned dependencies +mitmproxy <4; python_version < '3.6' # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines From 8e393a0b218b56cc8a70f7bda277969aa026790d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 12:29:51 +0200 Subject: [PATCH 230/568] Do not change the mitmproxy version for no-3.6 Python versions --- tests/requirements-py3.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index b67c08403..b425ce771 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,8 +1,9 @@ # Tests requirements attrs -dataclasses; python_version == '3.6' -mitmproxy >=4, <5; python_version < '3.6' # The latest version does not support some pinned dependencies -mitmproxy <4; python_version < '3.6' +dataclasses; python_version ==3.6 +mitmproxy; python_version >=3.7 +mitmproxy >=4, <5; python_version >=3.6, <3.7 +mitmproxy <4; python_version <3.6 # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines From b1de55d37d6a1b84e92df2d93d83e247e5d0e0d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 12:34:40 +0200 Subject: [PATCH 231/568] Fix marker syntax --- tests/requirements-py3.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index b425ce771..b51177abb 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,9 +1,9 @@ # Tests requirements attrs -dataclasses; python_version ==3.6 -mitmproxy; python_version >=3.7 -mitmproxy >=4, <5; python_version >=3.6, <3.7 -mitmproxy <4; python_version <3.6 +dataclasses; python_version == '3.6' +mitmproxy; python_version >= '3.7' +mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7' +mitmproxy < 4; python_version < '3.6' # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines From 125a058340cf77a70fe605b0317b3c517f637c81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 17:07:21 +0200 Subject: [PATCH 232/568] Do not let umask affect the permissions of startproject-generated files --- scrapy/utils/template.py | 6 +++-- tests/test_commands.py | 57 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 96ff4b09b..f068be737 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -12,10 +12,12 @@ def render_templatefile(path, **kwargs): content = string.Template(raw).substitute(**kwargs) render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path + + if path.endswith('.tmpl'): + os.rename(path, render_path) + with open(render_path, 'wb') as fp: fp.write(content.encode('utf8')) - if path.endswith('.tmpl'): - os.remove(path) CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') diff --git a/tests/test_commands.py b/tests/test_commands.py index 8938156fc..be88e3511 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -126,9 +126,13 @@ class StartprojectTest(ProjectTest): def get_permissions_dict(path, renamings=None, ignore=None): + + def get_permissions(path): + return oct(os.stat(path).st_mode) + renamings = renamings or tuple() permissions_dict = { - '.': os.stat(path).st_mode, + '.': get_permissions(path), } for root, dirs, files in os.walk(path): nodes = list(chain(dirs, files)) @@ -143,13 +147,15 @@ def get_permissions_dict(path, renamings=None, ignore=None): search_string, replacement ) - permissions = os.stat(absolute_path).st_mode + permissions = get_permissions(absolute_path) permissions_dict[relative_path] = permissions return permissions_dict class StartprojectTemplatesTest(ProjectTest): + maxDiff = None + def setUp(self): super().setUp() self.tmpl = join(self.temp_path, 'templates') @@ -311,6 +317,53 @@ class StartprojectTemplatesTest(ProjectTest): self.assertEqual(actual_permissions, expected_permissions) + def test_startproject_permissions_umask_022(self): + """Check that generated files have the right permissions when the + system uses a umask value that causes new files to have different + permissions than those from the template folder.""" + @contextmanager + def umask(new_mask): + cur_mask = os.umask(new_mask) + yield + os.umask(cur_mask) + + scrapy_path = scrapy.__path__[0] + project_template = os.path.join( + scrapy_path, + 'templates', + 'project' + ) + project_name = 'umaskproject' + renamings = ( + ('module', project_name), + ('.tmpl', ''), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + with umask(0o002): + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'startproject', + project_name, + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = os.path.join(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + self.assertEqual(actual_permissions, expected_permissions) + class CommandTest(ProjectTest): From 4c0afb606c59e6c8eabe4bf97cdb0494cf26dec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 17:45:26 +0200 Subject: [PATCH 233/568] Update permission expectations --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index be88e3511..55088c605 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -297,7 +297,7 @@ class StartprojectTemplatesTest(ProjectTest): path.mkdir(mode=permissions) else: path.touch(mode=permissions) - expected_permissions[node] = path.stat().st_mode + expected_permissions[node] = oct(path.stat().st_mode) process = subprocess.Popen( ( From 5f4df622a17b1d8f9c3b9b693cb2f7119ffec27f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 05:41:06 +0200 Subject: [PATCH 234/568] test_utils_iterators.py: support Windows the right way --- tests/__init__.py | 2 +- tests/test_utils_iterators.py | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 12ce79fa9..4253017fa 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -28,5 +28,5 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) - with open(path, 'rb') as f: + with open(path, 'rb', newline='') as f: return f.read() diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 298178f08..50190d4d1 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -7,9 +7,6 @@ from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata -FOOBAR_NL = "foo{}bar".format(os.linesep) - - class XmliterTestCase(unittest.TestCase): xmliter = staticmethod(xmliter) @@ -267,7 +264,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual(result, [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) # explicit type check cuz' we no like stinkin' autocasting! yarrr @@ -283,7 +280,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_quotechar(self): @@ -296,7 +293,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv1], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) response2 = TextResponse(url="http://example.com/", body=body2) @@ -305,7 +302,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv2], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_wrong_quotechar(self): @@ -327,7 +324,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_headers(self): @@ -353,7 +350,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_exception(self): From 24ba5a71aca9b368eee21bd7d9043a7d26dba403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 06:35:09 +0200 Subject: [PATCH 235/568] Maybe the problem is not in the code after all --- .gitattributes | 1 + tests/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..dfbdf4208 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/sample_data/** binary diff --git a/tests/__init__.py b/tests/__init__.py index 4253017fa..12ce79fa9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -28,5 +28,5 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) - with open(path, 'rb', newline='') as f: + with open(path, 'rb') as f: return f.read() From 65e0abaea5eddb1dbb28b2bca9cd00e9b9409471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 09:05:51 +0200 Subject: [PATCH 236/568] Document FEED_URI_PARAMS --- docs/topics/feed-exports.rst | 75 ++++++++++++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 37b7096f6..0f0f258dc 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -321,13 +321,14 @@ The following is a list of the accepted keys and the setting that is used as a fallback value if that key is not provided for a specific feed definition. * ``format``: the serialization format to be used for the feed. - See :ref:`topics-feed-format` for possible values. + See :ref:`topics-feed-format` for possible values. Mandatory, no fallback setting +* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` * ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING` * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` -* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` +* ``uri_params``: falls back to :setting:`FEED_URI_PARAMS` .. setting:: FEED_EXPORT_ENCODING @@ -500,7 +501,7 @@ generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created (e.g. ``2020-03-28T14-45-08.237134``) -* ``%(batch_id)d`` - gets replaced by the sequence number of the batch. +* ``%(batch_id)d`` - gets replaced by the 1-based sequence number of the batch. Use :ref:`printf-style string formatting ` to alter the number format. For example, to make the batch ID a 5-digit @@ -517,16 +518,74 @@ And your :command:`crawl` command line is:: The command line above can generate a directory tree like:: -->projectname --->dirname ---->1-filename2020-03-28T14-45-08.237134.json ---->2-filename2020-03-28T14-45-09.148903.json ---->3-filename2020-03-28T14-45-10.046092.json + ->projectname + -->dirname + --->1-filename2020-03-28T14-45-08.237134.json + --->2-filename2020-03-28T14-45-09.148903.json + --->3-filename2020-03-28T14-45-10.046092.json Where the first and second files contain exactly 100 items. The last one contains 100 items or fewer. +.. setting:: FEED_URI_PARAMS + +FEED_URI_PARAMS +--------------- + +Default: ``None`` + +A string with the import path of a function to set the parameters to apply with +:ref:`printf-style string formatting ` to the +feed URI. + +The function signature should be as follows: + +.. function:: uri_params(params, spider) + + Return a :class:`dict` of key-value pairs to apply to the feed URI using + :ref:`printf-style string formatting `. + + :param params: default key-value pairs + + Specifically: + + - ``batch_id``: ID of the file batch. See + :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + + If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id`` + is always ``1``. + + - ``batch_time``: UTC date and time, in ISO format with ``:`` + replaced with ``-``. + + See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + + - ``time``: ``batch_time``, with microseconds set to ``0``. + :type params: dict + + :param spider: source spider of the feed items + :type spider: scrapy.spiders.Spider + +For example, to include the :attr:`name ` of the +source spider in the feed URI: + +#. Define the following function somewhere in your project:: + + # myproject/utils.py + def uri_params(params, spider): + return {**params, 'spider_name': spider.name} + +#. Point :setting:`FEED_URI_PARAMS` to that function in your settings:: + + # myproject/settings.py + FEED_URI_PARAMS = 'myproject.utils.uri_params' + +#. Use ``%(spider_name)s`` in your feed URI:: + + scrapy crawl -o "%(spider_name)s.jl" + + .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ .. _botocore: https://github.com/boto/botocore From 756c368a6b0d2eef65f86f8418f9a7fbeff036c7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Aug 2020 22:09:24 +0500 Subject: [PATCH 237/568] Use a longer key in mitmproxy-ca.pem. --- tests/keys/mitmproxy-ca.pem | 78 +++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/tests/keys/mitmproxy-ca.pem b/tests/keys/mitmproxy-ca.pem index 08004feca..cdef75f99 100644 --- a/tests/keys/mitmproxy-ca.pem +++ b/tests/keys/mitmproxy-ca.pem @@ -1,32 +1,50 @@ ------BEGIN RSA PRIVATE KEY----- -MIICWwIBAAKBgQDKLbznLxS7HSWvrmGcvVS6eQvjEWD705/csvnk/WtqAPfQMJKt -auFBxzPt6RT60SHtj/2FKt2gqsiE6cNINxGN6fGYD7HtaM5HXRVPUKJaMipJwHha -QivjIZoueraY/MtlyCkpp6dmMnHEpGY7OzwMyh1eCBHQ2JYx6VEzbks9ewIDAQAB -AoGAMpS2ye/Rc+6a2xT5fskvRWe7PZe/d8E+IWz1cACmuuJ7HS7Jw3EV4esAZukF -QqrHnjOD7akHwYZ4nCgPnyWH0lLx/4TIXE5QeLPFrhKOsSLCyhlCwNVJAdcOrDol -Qh2694Dsd4gAy5o6TA02cBpqArnbAUERX46bHBZRA+ths8ECQQD6r1Ls+bTBR52w -T3rPPhYj7EsXp40MJt0pLf1kjf+EH1bxsUqnxLawwo/lLE9omU73DFnfrAflk2Ll -KUPCjjYpAkEAznchXk2ITeRcClrBNA+1Izpb5yG1qkfc79u/CEVDDOvt7RO/89Oj -58R3pKTyffoo34fBdJz8GYDsmOeiyJEjAwJANpSHrJrtlQt/tMyJQ6gT7/xZmSvc -1OF9U6L0wbj9AgpExtjAFWkKEdA6vj34iCChBb8FrmJpUb3WUWi7nReTiQJAFyIT -9Av93LRcd7CJezrTUdolF/WX9DdPEvTtJ5ETHSyGIQ0Yccph0AMcYK82mFTiJYGB -dH5uZLEkUVGK1KwmXwJAGWLdYiQyQitRWdoURcLb4OZ2gF3+7PASgFilI8YuoYhn -Rl2Va3UtErPKJeMg2dTH18PuXykQMsQR1+rPxf1WSA== ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCYp6U4G9YWITYB +/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7ZDiT +NUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMgz1BJ +u6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniIggUH +JrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE6HFB +eIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/0zls +90iyQ3E/AgMBAAECggEBAJA1dyAdM85uC04vKVNUJM1GDp0xS+0syBReJaKRI3nJ +epoCj+RqxGag1pdaYLI0G84NTPqECz9LOyLdqpPgEfKRIxWlf9oWmSnfnXskArd8 +VfVcWYl6tEPv1TToTZIBmCbYLBFVbLxG/GrbK6uokdhUsqbdXwEKok2IEaSTRlDn +v8BVXte00d9VEKKpmI6EY3f45uPQPHuJNcitP2HGW1mT/C6XoZR6wj+VvoRgUGQT +I7PuktbYpQlLV+oX0uZz9frPGhjydUq0Jti5v3QAJEb+7D0cKrkZW+7fYDx4YkRU +oDiuWEyO2kfpff52Qxs+xUXMiAyw6/8+TamKoAi1TIECgYEAyAzoztW6W4CjL2au +/hN5VmbAvuBxq1m1G5KgXM1myX9V2CgH6OKwzJQNSCEfKMNOjqxB99T7C3tMCjgG +gmbUzylTeciQFF+crrl2Rn/6qZS9dCo1hagb3K5eXMhLXoP425Y4sypNPPqULhPn +YrUDFNAf89rRLqP1KMPLZ+uO7EECgYEAw1lWPxGV+X85iQxYN9xoX85htfJSBXTf +dLirQ4bkykOxSA6ZzFuhDO/G373Q1rze4tmEO790uOCeaiXGgeWC1A+2PMO957i5 +9FqhDIkmerfdIttdEUMM9rQwuTcLnixGZkT5GHDzjtNinaIVB+pv7twRAESqN9dC +QXh7IF7g/X8CgYBMhQOX+hCqZ24D95cAAJrs/ajEWj2geVPZFCDa3oZulJJVeBpu +bieKWScra9/rS6mE0Ub6cTEFl0fisMNspcDI7NnNP3Y9FMVt3+rp1JIgw5AkGvEW +CtN9egUGIGcT5A8Qj0lo3slkhcSgS2S6UNq431MZh51z5askyJ/JREULAQKBgFrR +OatwfYzUfOcd+hVePpfr1rlDwqYOw6P8BoMKP2tZNR4Oy6maH7Fn98kk8eYjQGuu +PC+avqUEqCEpFrRlAwGbnFl7ltoXozvatmyhhmYe/Iur+ASCa5B2DQDOenQ6mTAK +eNPIDzMjSwGFzMk1UHx3it/ZDFmRlZfibzuJYIf5AoGBAIaPHk4qadK/XpcD4Wwx +BOsDEIz27DGWdwWfd5r3EcV4zX/wNzH0G1Z8eydNjUqKzufMZgFwpcTu0Evesl1/ +B8kC8sLHxQoG5SvBu4dBxMwKIU9O9uFnX5SUYZUDpCtUYyZ+GtGom41Jwg5ENrwy +HzPh2taMnCA0h1fNLFFBkw88 +-----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICnzCCAgigAwIBAgIGDI2K/EOjMA0GCSqGSIb3DQEBBQUAMCgxEjAQBgNVBAMT -CW1pdG1wcm94eTESMBAGA1UEChMJbWl0bXByb3h5MB4XDTEzMDkyNjE0MzYxMVoX -DTE1MDkxNjE0MzYxMVowKDESMBAGA1UEAxMJbWl0bXByb3h5MRIwEAYDVQQKEwlt -aXRtcHJveHkwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMotvOcvFLsdJa+u -YZy9VLp5C+MRYPvTn9yy+eT9a2oA99Awkq1q4UHHM+3pFPrRIe2P/YUq3aCqyITp -w0g3EY3p8ZgPse1ozkddFU9QoloyKknAeFpCK+Mhmi56tpj8y2XIKSmnp2YyccSk -Zjs7PAzKHV4IEdDYljHpUTNuSz17AgMBAAGjgdMwgdAwDwYDVR0TAQH/BAUwAwEB -/zAUBglghkgBhvhCAQEBAf8EBAMCAgQwewYDVR0lAQH/BHEwbwYIKwYBBQUHAwEG -CCsGAQUFBwMCBggrBgEFBQcDBAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQB -gjcCARYGCisGAQQBgjcKAwEGCisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG -+EIEATALBgNVHQ8EBAMCAQYwHQYDVR0OBBYEFJBEfawVwhEHHW6rS8nvZFlJ582n -MA0GCSqGSIb3DQEBBQUAA4GBAHGl28Ip2CWS/MibCaFztLDxGiMBT4MW2yI2hf3D -y9g1o7ra/fSEFdIc849xXyCsGWSkMsbDML272rCH4K73MUBxxkJm46AIyRVH1z2Z -e96u4py1wNT8cznY15phr8pn36snlaHaYa+JcwGINMdSOk1VPHv6gqSC/vgUCgF1 -n95u +MIIDoTCCAomgAwIBAgIGDodLQx9+MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM +CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTIwMDgxMjE3MDMyNloX +DTIzMDgxNDE3MDMyNlowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt +aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYp6U4G9YW +ITYB/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7 +ZDiTNUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMg +z1BJu6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniI +ggUHJrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE +6HFBeIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/ +0zls90iyQ3E/AgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC +AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD +BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG +CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC +AQYwHQYDVR0OBBYEFBCsLPpFz3l9rOOfGmfs+VRc3jhJMA0GCSqGSIb3DQEBCwUA +A4IBAQADTpA15na6U5qqDCe0rr39fkS1/dY804Xnz7g/L3AsxPE1KOMijuJa8sKd +kKwba1173FwMupfK39zY8jUxL8Qprdi92RO6CpoFUsL/icpA///lYhzUSqt32qwe +gRNW3mtYBimOk6KH1NOfQnJolWpJh+g1OEsitQKEeKwIn5Hz+8/yS5tbwLgdnMlY +1/it1H70JSdE7nfJueqN4cFfBsm6XaHZzacJJmN7WP88fd+zztnSQsBFbLlnjnqj +envCDIwCrMywKNMqEBMwmBEGSAF47fVNYj6KzDAtMvBdDkYaHWpBf4tnFfk6v0wj +wiKjdLjCmJgjGAQjRw5VYJ8JI0XO -----END CERTIFICATE----- From af73f141b23aabbef208a12fe95ae63c27105fda Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 16 Aug 2020 11:26:10 +0530 Subject: [PATCH 238/568] refactor: move all http2 tests in separate files --- tests/test_download_handlers_http2.py | 158 ++++++++++++++++++++++++++ tests/test_downloader_handlers.py | 141 +---------------------- 2 files changed, 160 insertions(+), 139 deletions(-) create mode 100644 tests/test_download_handlers_http2.py diff --git a/tests/test_download_handlers_http2.py b/tests/test_download_handlers_http2.py new file mode 100644 index 000000000..583dc1d17 --- /dev/null +++ b/tests/test_download_handlers_http2.py @@ -0,0 +1,158 @@ +from unittest import mock + +from twisted.internet import defer, error, reactor +from twisted.trial import unittest +from twisted.web import server + +from scrapy.core.downloader.handlers.http2 import H2DownloadHandler +from scrapy.http import Request +from scrapy.spiders import Spider +from scrapy.utils.misc import create_instance +from scrapy.utils.test import get_crawler +from tests.mockserver import ssl_context_factory +from tests.test_downloader_handlers import ( + Https11TestCase, Https11CustomCiphers, + Http11MockServerTestCase, Http11ProxyTestCase, + UriResource +) + + +class Https2TestCase(Https11TestCase): + scheme = 'https' + download_handler_cls = H2DownloadHandler + HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + + @defer.inlineCallbacks + def test_download_with_maxsize_very_large_file(self): + with mock.patch('scrapy.core.http2.stream.logger') as logger: + request = Request(self.getURL('largechunkedfile')) + + def check(logger): + logger.error.assert_called_once_with(mock.ANY) + + d = self.download_request(request, Spider('foo', download_maxsize=1500)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + # As the error message is logged in the dataReceived callback, we + # have to give a bit of time to the reactor to process the queue + # after closing the connection. + d = defer.Deferred() + d.addCallback(check) + reactor.callLater(.1, d.callback, logger) + yield d + + def test_download_broken_content_cause_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_cause_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + +class Https2WrongHostnameTestCase(Https2TestCase): + tls_log_message = ( + 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' + 'subject "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' + ) + + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = 'keys/example-com.key.pem' + certfile = 'keys/example-com.cert.pem' + + +class Https2InvalidDNSId(Https2TestCase): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super(Https2InvalidDNSId, self).setUp() + self.host = '127.0.0.1' + + +class Https2InvalidDNSPattern(Https2TestCase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = 'keys/localhost.ip.key' + certfile = 'keys/localhost.ip.crt' + + def setUp(self): + try: + from service_identity.exceptions import CertificateError # noqa: F401 + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") + self.tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' + 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + ) + super(Https2InvalidDNSPattern, self).setUp() + + +class Https2CustomCiphers(Https11CustomCiphers): + scheme = 'https' + download_handler_cls = H2DownloadHandler + + +class Http2MockServerTestCase(Http11MockServerTestCase): + """HTTP 2.0 test case with MockServer""" + + +class Https2ProxyTestCase(Http11ProxyTestCase): + # only used for HTTPS tests + keyfile = 'keys/localhost.key' + certfile = 'keys/localhost.crt' + + scheme = 'https' + host = u'127.0.0.1' + + download_handler_cls = H2DownloadHandler + expected_http_proxy_request_body = b'/' + + def setUp(self): + site = server.Site(UriResource(), timeout=None) + self.port = reactor.listenSSL( + 0, site, + ssl_context_factory(self.keyfile, self.certfile), + interface=self.host + ) + self.portno = self.port.getHost().port + self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) + self.download_request = self.download_handler.download_request + + def getURL(self, path): + return f"{self.scheme}://{self.host}:{self.portno}/{path}" + + def test_download_with_proxy_https_noconnect(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'/') + + http_proxy = '%s?noconnect' % self.getURL('') + request = Request('https://example.com', meta={'proxy': http_proxy}) + with self.assertWarnsRegex( + Warning, + r'Using HTTPS proxies in the noconnect mode is not supported by the ' + r'downloader handler.' + ): + return self.download_request(request, Spider('foo')).addCallback(_test) + + @defer.inlineCallbacks + def test_download_with_proxy_https_timeout(self): + with self.assertRaises(NotImplementedError): + yield super(Https2ProxyTestCase, self).test_download_with_proxy_https_timeout() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 1cedf6b10..8b2b2c32c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -23,7 +23,6 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler -from scrapy.core.downloader.handlers.http2 import H2DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Headers, Request @@ -315,8 +314,8 @@ class HttpTestCase(unittest.TestCase): host = self.host + ':' + str(self.portno) def _test(response): - self.assertEqual(response.body, to_bytes(host)) - self.assertEqual(request.headers.get('Host'), to_bytes(host)) + self.assertEqual(response.body, host.encode()) + self.assertEqual(request.headers.get('Host'), host.encode()) request = Request(self.getURL('host'), headers={'Host': host}) return self.download_request(request, Spider('foo')).addCallback(_test) @@ -522,49 +521,6 @@ class Https11TestCase(Http11TestCase): yield download_handler.close() -class Https2TestCase(Https11TestCase): - scheme = 'https' - download_handler_cls = H2DownloadHandler - HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): - with mock.patch('scrapy.core.http2.stream.logger') as logger: - request = Request(self.getURL('largechunkedfile')) - - def check(logger): - logger.error.assert_called_once_with(mock.ANY) - - d = self.download_request(request, Spider('foo', download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - # As the error message is logged in the dataReceived callback, we - # have to give a bit of time to the reactor to process the queue - # after closing the connection. - d = defer.Deferred() - d.addCallback(check) - reactor.callLater(.1, d.callback, logger) - yield d - - def test_download_broken_content_cause_data_loss(self, url='broken'): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_chunked_content_cause_data_loss(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_content_allow_data_loss(self, url='broken'): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_chunked_content_allow_data_loss(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_chunked_content_allow_data_loss_via_setting(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - class Https11WrongHostnameTestCase(Http11TestCase): scheme = 'https' @@ -577,23 +533,6 @@ class Https11WrongHostnameTestCase(Http11TestCase): certfile = 'keys/example-com.cert.pem' -class Https2WrongHostnameTestCase(Https2TestCase): - tls_log_message = ( - 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' - 'subject "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' - ) - - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = 'keys/example-com.key.pem' - certfile = 'keys/example-com.cert.pem' - - class Https11InvalidDNSId(Https11TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" @@ -602,14 +541,6 @@ class Https11InvalidDNSId(Https11TestCase): self.host = '127.0.0.1' -class Https2InvalidDNSId(Https2TestCase): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super(Https2InvalidDNSId, self).setUp() - self.host = '127.0.0.1' - - class Https11InvalidDNSPattern(Https11TestCase): """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" @@ -628,24 +559,6 @@ class Https11InvalidDNSPattern(Https11TestCase): super(Https11InvalidDNSPattern, self).setUp() -class Https2InvalidDNSPattern(Https2TestCase): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = 'keys/localhost.ip.key' - certfile = 'keys/localhost.ip.crt' - - def setUp(self): - try: - from service_identity.exceptions import CertificateError # noqa: F401 - except ImportError: - raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' - 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' - ) - super(Https2InvalidDNSPattern, self).setUp() - - class Https11CustomCiphers(unittest.TestCase): scheme = 'https' download_handler_cls = HTTP11DownloadHandler @@ -686,11 +599,6 @@ class Https11CustomCiphers(unittest.TestCase): return d -class Https2CustomCiphers(Https11CustomCiphers): - scheme = 'https' - download_handler_cls = H2DownloadHandler - - class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" @@ -742,10 +650,6 @@ class Http11MockServerTestCase(unittest.TestCase): self.assertTrue(reason, 'finished') -class Http2MockServerTestCase(Http11MockServerTestCase): - """HTTP 2.0 test case with MockServer""" - - class UriResource(resource.Resource): """Return the full uri that was requested""" @@ -837,47 +741,6 @@ class Http11ProxyTestCase(HttpProxyTestCase): self.assertIn(domain, timeout.osError) -class Https2ProxyTestCase(Http11ProxyTestCase): - # only used for HTTPS tests - keyfile = 'keys/localhost.key' - certfile = 'keys/localhost.crt' - - scheme = 'https' - host = u'127.0.0.1' - - download_handler_cls = H2DownloadHandler - expected_http_proxy_request_body = b'/' - - def setUp(self): - site = server.Site(UriResource(), timeout=None) - self.port = reactor.listenSSL( - 0, site, - ssl_context_factory(self.keyfile, self.certfile), - interface=self.host - ) - self.portno = self.port.getHost().port - self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) - self.download_request = self.download_handler.download_request - - def getURL(self, path): - return f"{self.scheme}://{self.host}:{self.portno}/{path}" - - def test_download_with_proxy_https_noconnect(self): - def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'/') - - http_proxy = '%s?noconnect' % self.getURL('') - request = Request('https://example.com', meta={'proxy': http_proxy}) - with self.assertWarnsRegex( - Warning, - r'Using HTTPS proxies in the noconnect mode is not supported by the ' - r'downloader handler.' - ): - return self.download_request(request, Spider('foo')).addCallback(_test) - - class HttpDownloadHandlerMock: def __init__(self, *args, **kwargs): From f9f008e935c5c892c2839d354ded3ee213057d9e Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 16 Aug 2020 17:04:40 +0530 Subject: [PATCH 239/568] test: add typing-extensions --- .travis.yml | 2 +- setup.py | 1 + tox.ini | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e273e358d..0b55cda19 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ dist: xenial branches: only: - master - - http2 # Remove once merged into master + - http2 # ToDo: Remove once merged into master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: diff --git a/setup.py b/setup.py index 7f4ff0095..c8733ae96 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.2.0', + 'typing-extensions>=3.7', ] extras_require = {} diff --git a/tox.ini b/tox.ini index e8fdbd85d..3bf2224ec 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,7 @@ deps = -ctests/constraints.txt -rtests/requirements-py3.txt # Extras + Twisted[http2]>=17.9.0 boto3>=1.13.0 botocore>=1.3.23 Pillow>=3.4.2 @@ -64,6 +65,7 @@ deps = -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 + h2==3.2.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 @@ -72,12 +74,12 @@ deps = queuelib==1.4.2 service_identity==16.0.0 Twisted[http2]==17.9.0 + typing-extensions==3.7 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt # Extras botocore==1.3.23 - h2==3.2.0 google-cloud-storage==1.29.0 Pillow==3.4.2 From 38d361792c02ae2b25323258d070c04d8906495a Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 16 Aug 2020 17:55:16 +0530 Subject: [PATCH 240/568] fix: typing & pylint errors - Ignore typing check for http2 test files --- scrapy/core/downloader/handlers/http2.py | 4 ++-- scrapy/core/http2/protocol.py | 2 +- setup.cfg | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 650af9778..f2ed40f9b 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -71,8 +71,8 @@ class ScrapyH2Agent: def __init__( self, context_factory, - connect_timeout=10, - bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None, + pool: H2ConnectionPool, + connect_timeout=10, bind_address: Optional[bytes] = None, crawler=None ) -> None: self._context_factory = context_factory diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 1ce8b6548..fee391af6 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -323,7 +323,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) elif isinstance(event, UnknownFrameReceived): - logger.debug(f'UnknownFrameReceived: frame={event.frame}') + logger.debug('UnknownFrameReceived: frame={}'.format(event.frame)) # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated) -> None: diff --git a/setup.cfg b/setup.cfg index f8e7c0c91..8b70a0e60 100644 --- a/setup.cfg +++ b/setup.cfg @@ -100,6 +100,9 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True +[mypy-tests.test_download_handlers_http2] +ignore_errors = True + [mypy-tests.test_engine] ignore_errors = True @@ -109,6 +112,9 @@ ignore_errors = True [mypy-tests.test_http_request] ignore_errors = True +[mypy-tests.test_http2_client_protocol] +ignore_errors = True + [mypy-tests.test_linkextractors] ignore_errors = True From 75fe3d13657e11bdabb9b26b452c6cdacecffec7 Mon Sep 17 00:00:00 2001 From: adityaa30 Date: Mon, 17 Aug 2020 03:47:17 +0530 Subject: [PATCH 241/568] fix: increase timeout to 0.5 seconds - In Windows specifically the reactor was left unclean by the HostnameEndpoint due to the tearDown method of test_downloader_handlers.py::HttpTestCase due to which the following 2 tests were failing: 1. test_timeout_download_from_spider_server_hangs 2. test_timeout_download_from_spider_nodata_rcvd - Increasing the timeout fixed the test (in local) --- setup.cfg | 2 +- tests/test_downloader_handlers.py | 4 ++-- ...ad_handlers_http2.py => test_downloader_handlers_http2.py} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename tests/{test_download_handlers_http2.py => test_downloader_handlers_http2.py} (100%) diff --git a/setup.cfg b/setup.cfg index 8b70a0e60..5b267c295 100644 --- a/setup.cfg +++ b/setup.cfg @@ -100,7 +100,7 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_download_handlers_http2] +[mypy-tests.test_downloader_handlers_http2] ignore_errors = True [mypy-tests.test_engine] diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index ecac47d90..2b3fa2aca 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -287,7 +287,7 @@ class HttpTestCase(unittest.TestCase): def test_timeout_download_from_spider_nodata_rcvd(self): # client connects but no data is received spider = Spider('foo') - meta = {'download_timeout': 0.2} + meta = {'download_timeout': 0.5} request = Request(self.getURL('wait'), meta=meta) d = self.download_request(request, spider) yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) @@ -296,7 +296,7 @@ class HttpTestCase(unittest.TestCase): def test_timeout_download_from_spider_server_hangs(self): # client connects, server send headers and some body bytes but hangs spider = Spider('foo') - meta = {'download_timeout': 0.2} + meta = {'download_timeout': 0.5} request = Request(self.getURL('hang-after-headers'), meta=meta) d = self.download_request(request, spider) yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) diff --git a/tests/test_download_handlers_http2.py b/tests/test_downloader_handlers_http2.py similarity index 100% rename from tests/test_download_handlers_http2.py rename to tests/test_downloader_handlers_http2.py From 2aa4f3cbf96ad55aa4a1fa064758338daf16b110 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 17 Aug 2020 05:39:59 -0300 Subject: [PATCH 242/568] Conditional request attribute binding for responses (#4632) --- docs/topics/signals.rst | 5 + scrapy/core/engine.py | 13 +- scrapy/core/scraper.py | 62 ++++---- setup.cfg | 3 + tests/test_request_attribute_binding.py | 202 ++++++++++++++++++++++++ 5 files changed, 250 insertions(+), 35 deletions(-) create mode 100644 tests/test_request_attribute_binding.py diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 255ba9d3f..1d99d8c28 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -423,6 +423,11 @@ response_received :param spider: the spider for which the response is intended :type spider: :class:`~scrapy.spiders.Spider` object +.. note:: The ``request`` argument might not contain the original request that + reached the downloader, if a :ref:`topics-downloader-middleware` modifies + the :class:`~scrapy.http.Response` object and sets a specific ``request`` + attribute. + response_downloaded ~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 86a6abb23..5e0dfe37c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -243,12 +243,17 @@ class ExecutionEngine: % (type(response), response) ) if isinstance(response, Response): - response.request = request # tie request to response received - logkws = self.logformatter.crawled(request, response, spider) + if response.request is None: + response.request = request + logkws = self.logformatter.crawled(response.request, response, spider) if logkws is not None: logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) - self.signals.send_catch_log(signals.response_received, - response=response, request=request, spider=spider) + self.signals.send_catch_log( + signal=signals.response_received, + response=response, + request=response.request, + spider=spider, + ) return response def _on_complete(_): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 1ef0790a9..20bdb22a1 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -12,7 +12,7 @@ from scrapy import signals from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response -from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel +from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output @@ -120,40 +120,40 @@ class Scraper: response, request, deferred = slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, response, request, spider): - """Handle the downloaded response or failure through the spider - callback/errback""" - if not isinstance(response, (Response, Failure)): - raise TypeError( - "Incorrect type: expected Response or Failure, got %s: %r" - % (type(response), response) - ) - - dfd = self._scrape2(response, request, spider) # returns spider's processed output - dfd.addErrback(self.handle_spider_error, request, response, spider) - dfd.addCallback(self.handle_spider_output, request, response, spider) + def _scrape(self, result, request, spider): + """ + Handle the downloaded response or failure through the spider callback/errback + """ + if not isinstance(result, (Response, Failure)): + raise TypeError("Incorrect type: expected Response or Failure, got %s: %r" % (type(result), result)) + dfd = self._scrape2(result, request, spider) # returns spider's processed output + dfd.addErrback(self.handle_spider_error, request, result, spider) + dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, request_result, request, spider): - """Handle the different cases of request's result been a Response or a - Failure""" - if not isinstance(request_result, Failure): - return self.spidermw.scrape_response( - self.call_spider, request_result, request, spider) - else: - dfd = self.call_spider(request_result, request, spider) - return dfd.addErrback( - self._log_download_errors, request_result, request, spider) + def _scrape2(self, result, request, spider): + """ + Handle the different cases of request's result been a Response or a Failure + """ + if isinstance(result, Response): + return self.spidermw.scrape_response(self.call_spider, result, request, spider) + else: # result is a Failure + dfd = self.call_spider(result, request, spider) + return dfd.addErrback(self._log_download_errors, result, request, spider) def call_spider(self, result, request, spider): - result.request = request - dfd = defer_result(result) - callback = request.callback or spider._parse - warn_on_generator_with_return_value(spider, callback) - warn_on_generator_with_return_value(spider, request.errback) - dfd.addCallbacks(callback=callback, - errback=request.errback, - callbackKeywords=request.cb_kwargs) + if isinstance(result, Response): + if getattr(result, "request", None) is None: + result.request = request + callback = result.request.callback or spider._parse + warn_on_generator_with_return_value(spider, callback) + dfd = defer_succeed(result) + dfd.addCallback(callback, **result.request.cb_kwargs) + else: # result is a Failure + result.request = request + warn_on_generator_with_return_value(spider, request.errback) + dfd = defer_fail(result) + dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) def handle_spider_error(self, _failure, request, response, spider): diff --git a/setup.cfg b/setup.cfg index f8e7c0c91..3a624ec94 100644 --- a/setup.cfg +++ b/setup.cfg @@ -130,6 +130,9 @@ ignore_errors = True [mypy-tests.test_pipelines] ignore_errors = True +[mypy-tests.test_request_attribute_binding] +ignore_errors = True + [mypy-tests.test_request_cb_kwargs] ignore_errors = True diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py new file mode 100644 index 000000000..b60b7c579 --- /dev/null +++ b/tests/test_request_attribute_binding.py @@ -0,0 +1,202 @@ +from twisted.internet import defer +from twisted.trial.unittest import TestCase + +from scrapy import Request, signals +from scrapy.crawler import CrawlerRunner +from scrapy.http.response import Response + +from testfixtures import LogCapture + +from tests.mockserver import MockServer +from tests.spiders import SingleRequestSpider + + +OVERRIDEN_URL = "https://example.org" + + +class ProcessResponseMiddleware: + def process_response(self, request, response, spider): + return response.replace(request=Request(OVERRIDEN_URL)) + + +class RaiseExceptionRequestMiddleware: + def process_request(self, request, spider): + 1 / 0 + return request + + +class CatchExceptionOverrideRequestMiddleware: + def process_exception(self, request, exception, spider): + return Response( + url="http://localhost/", + body=b"Caught " + exception.__class__.__name__.encode("utf-8"), + request=Request(OVERRIDEN_URL), + ) + + +class CatchExceptionDoNotOverrideRequestMiddleware: + def process_exception(self, request, exception, spider): + return Response( + url="http://localhost/", + body=b"Caught " + exception.__class__.__name__.encode("utf-8"), + ) + + +class AlternativeCallbacksSpider(SingleRequestSpider): + name = "alternative_callbacks_spider" + + def alt_callback(self, response, foo=None): + self.logger.info("alt_callback was invoked with foo=%s", foo) + + +class AlternativeCallbacksMiddleware: + def process_response(self, request, response, spider): + new_request = request.replace( + url=OVERRIDEN_URL, + callback=spider.alt_callback, + cb_kwargs={"foo": "bar"}, + ) + return response.replace(request=new_request) + + +class CrawlTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_response_200(self): + url = self.mockserver.url("/status?n=200") + crawler = CrawlerRunner().create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.request.url, url) + + @defer.inlineCallbacks + def test_response_error(self): + for status in ("404", "500"): + url = self.mockserver.url("/status?n={}".format(status)) + crawler = CrawlerRunner().create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + failure = crawler.spider.meta["failure"] + response = failure.value.response + self.assertEqual(failure.request.url, url) + self.assertEqual(response.request.url, url) + + @defer.inlineCallbacks + def test_downloader_middleware_raise_exception(self): + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".RaiseExceptionRequestMiddleware": 590, + }, + }) + crawler = runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + failure = crawler.spider.meta["failure"] + self.assertEqual(failure.request.url, url) + self.assertIsInstance(failure.value, ZeroDivisionError) + + @defer.inlineCallbacks + def test_downloader_middleware_override_request_in_process_response(self): + """ + Downloader middleware which returns a response with an specific 'request' attribute. + + * The spider callback should receive the overriden response.request + * Handlers listening to the response_received signal should receive the overriden response.request + * The "crawled" log message should show the overriden response.request + """ + signal_params = {} + + def signal_handler(response, request, spider): + signal_params["response"] = response + signal_params["request"] = request + + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".ProcessResponseMiddleware": 595, + } + }) + crawler = runner.create_crawler(SingleRequestSpider) + crawler.signals.connect(signal_handler, signal=signals.response_received) + + with LogCapture() as log: + yield crawler.crawl(seed=url, mockserver=self.mockserver) + + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.request.url, OVERRIDEN_URL) + + self.assertEqual(signal_params["response"].url, url) + self.assertEqual(signal_params["request"].url, OVERRIDEN_URL) + + log.check_present( + ("scrapy.core.engine", "DEBUG", "Crawled (200) (referer: None)".format(OVERRIDEN_URL)), + ) + + @defer.inlineCallbacks + def test_downloader_middleware_override_in_process_exception(self): + """ + An exception is raised but caught by the next middleware, which + returns a Response with a specific 'request' attribute. + + The spider callback should receive the overriden response.request + """ + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".RaiseExceptionRequestMiddleware": 590, + __name__ + ".CatchExceptionOverrideRequestMiddleware": 595, + }, + }) + crawler = runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.body, b"Caught ZeroDivisionError") + self.assertEqual(response.request.url, OVERRIDEN_URL) + + @defer.inlineCallbacks + def test_downloader_middleware_do_not_override_in_process_exception(self): + """ + An exception is raised but caught by the next middleware, which + returns a Response without a specific 'request' attribute. + + The spider callback should receive the original response.request + """ + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".RaiseExceptionRequestMiddleware": 590, + __name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595, + }, + }) + crawler = runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.body, b"Caught ZeroDivisionError") + self.assertEqual(response.request.url, url) + + @defer.inlineCallbacks + def test_downloader_middleware_alternative_callback(self): + """ + Downloader middleware which returns a response with a + specific 'request' attribute, with an alternative callback + """ + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".AlternativeCallbacksMiddleware": 595, + } + }) + crawler = runner.create_crawler(AlternativeCallbacksSpider) + + with LogCapture() as log: + url = self.mockserver.url("/status?n=200") + yield crawler.crawl(seed=url, mockserver=self.mockserver) + + log.check_present( + ("alternative_callbacks_spider", "INFO", "alt_callback was invoked with foo=bar"), + ) From a8e08d51cd50e8f0b58a60992d310e56d4f71641 Mon Sep 17 00:00:00 2001 From: Ajay Mittur Date: Mon, 17 Aug 2020 14:15:52 +0530 Subject: [PATCH 243/568] Check if file is already present on running `scrapy genspider` and terminate if so (#4623) --- scrapy/commands/genspider.py | 41 ++++++++++++++++------ tests/test_commands.py | 67 +++++++++++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 4c7548e9c..74a077d1b 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -66,16 +66,9 @@ class Command(ScrapyCommand): print("Cannot create a spider with the same name as your project") return - try: - spidercls = self.crawler_process.spider_loader.load(name) - except KeyError: - pass - else: - # if spider already exists and not --force then halt - if not opts.force: - print("Spider %r already exists in module:" % name) - print(" %s" % spidercls.__module__) - return + if not opts.force and self._spider_exists(name): + return + template_file = self._find_template(opts.template) if template_file: self._genspider(module, name, domain, opts.template, template_file) @@ -119,6 +112,34 @@ class Command(ScrapyCommand): if filename.endswith('.tmpl'): print(" %s" % splitext(filename)[0]) + def _spider_exists(self, name): + if not self.settings.get('NEWSPIDER_MODULE'): + # if run as a standalone command and file with same filename already exists + if exists(name + ".py"): + print("%s already exists" % (abspath(name + ".py"))) + return True + return False + + try: + spidercls = self.crawler_process.spider_loader.load(name) + except KeyError: + pass + else: + # if spider with same name exists + print("Spider %r already exists in module:" % name) + print(" %s" % spidercls.__module__) + return True + + # a file with the same name exists in the target directory + spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) + spiders_dir = dirname(spiders_module.__file__) + spiders_dir_abs = abspath(spiders_dir) + if exists(join(spiders_dir_abs, name + ".py")): + print("%s already exists" % (join(spiders_dir_abs, (name + ".py")))) + return True + + return False + @property def templates_dir(self): return join( diff --git a/tests/test_commands.py b/tests/test_commands.py index 8938156fc..109f006a8 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -8,7 +8,7 @@ import sys import tempfile from contextlib import contextmanager from itertools import chain -from os.path import exists, join, abspath +from os.path import exists, join, abspath, getmtime from pathlib import Path from shutil import rmtree, copytree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION @@ -337,8 +337,11 @@ class GenspiderCommandTest(CommandTest): p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) + modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Spider %r already exists in module" % spname, out) + modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) + self.assertEqual(modify_time_after, modify_time_before) def test_template_basic(self): self.test_template('basic') @@ -360,6 +363,40 @@ class GenspiderCommandTest(CommandTest): self.assertEqual(2, self.call('genspider', self.project_name)) assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name)) + def test_same_filename_as_existing_spider(self, force=False): + file_name = 'example' + file_path = join(self.proj_mod_path, 'spiders', '%s.py' % file_name) + self.assertEqual(0, self.call('genspider', file_name, 'example.com')) + assert exists(file_path) + + # change name of spider but not its file name + with open(file_path, 'r+') as spider_file: + file_data = spider_file.read() + file_data = file_data.replace("name = \'example\'", "name = \'renamed\'") + spider_file.seek(0) + spider_file.write(file_data) + spider_file.truncate() + modify_time_before = getmtime(file_path) + file_contents_before = file_data + + if force: + p, out, err = self.proc('genspider', '--force', file_name, 'example.com') + self.assertIn("Created spider %r using template \'basic\' in module" % file_name, out) + modify_time_after = getmtime(file_path) + self.assertNotEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertNotEqual(file_contents_after, file_contents_before) + else: + p, out, err = self.proc('genspider', file_name, 'example.com') + self.assertIn("%s already exists" % (file_path), out) + modify_time_after = getmtime(file_path) + self.assertEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertEqual(file_contents_after, file_contents_before) + + def test_same_filename_as_existing_spider_force(self): + self.test_same_filename_as_existing_spider(force=True) + class GenspiderStandaloneCommandTest(ProjectTest): @@ -367,6 +404,34 @@ class GenspiderStandaloneCommandTest(ProjectTest): self.call('genspider', 'example', 'example.com') assert exists(join(self.temp_path, 'example.py')) + def test_same_name_as_existing_file(self, force=False): + file_name = 'example' + file_path = join(self.temp_path, file_name + '.py') + p, out, err = self.proc('genspider', file_name, 'example.com') + self.assertIn("Created spider %r using template \'basic\' " % file_name, out) + assert exists(file_path) + modify_time_before = getmtime(file_path) + file_contents_before = open(file_path, 'r').read() + + if force: + # use different template to ensure contents were changed + p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com') + self.assertIn("Created spider %r using template \'crawl\' " % file_name, out) + modify_time_after = getmtime(file_path) + self.assertNotEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertNotEqual(file_contents_after, file_contents_before) + else: + p, out, err = self.proc('genspider', file_name, 'example.com') + self.assertIn("%s already exists" % join(self.temp_path, file_name + ".py"), out) + modify_time_after = getmtime(file_path) + self.assertEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertEqual(file_contents_after, file_contents_before) + + def test_same_name_as_existing_file_force(self): + self.test_same_name_as_existing_file(force=True) + class MiscCommandsTest(CommandTest): From 55edf8d3b8885541cdbf9d1c62d9a6bbf634e2a0 Mon Sep 17 00:00:00 2001 From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com> Date: Mon, 17 Aug 2020 18:50:52 +1000 Subject: [PATCH 244/568] Add typing hint to httpcache downloadermiddlewares (#4243) --- scrapy/downloadermiddlewares/httpcache.py | 41 ++++++++++++++++------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 6db57bd8b..62f1c3a29 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,4 +1,5 @@ from email.utils import formatdate +from typing import Optional, Type, TypeVar from twisted.internet import defer from twisted.internet.error import ( @@ -13,10 +14,19 @@ from twisted.internet.error import ( from twisted.web.client import ResponseFailed from scrapy import signals +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.http.request import Request +from scrapy.http.response import Response +from scrapy.settings import Settings +from scrapy.spiders import Spider +from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object +HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware") + + class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, @@ -24,7 +34,7 @@ class HttpCacheMiddleware: ConnectionLost, TCPTimedOutError, ResponseFailed, IOError) - def __init__(self, settings, stats): + def __init__(self, settings: Settings, stats: StatsCollector) -> None: if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings) @@ -33,26 +43,26 @@ class HttpCacheMiddleware: self.stats = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls: Type[HttpCacheMiddlewareTV], crawler: Crawler) -> HttpCacheMiddlewareTV: o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self.storage.open_spider(spider) - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> None: self.storage.close_spider(spider) - def process_request(self, request, spider): + def process_request(self, request: Request, spider: Spider) -> Optional[Response]: if request.meta.get('dont_cache', False): - return + return None # Skip uncacheable requests if not self.policy.should_cache_request(request): request.meta['_dont_cache'] = True # flag as uncacheable - return + return None # Look for cached response and check if expired cachedresponse = self.storage.retrieve_response(spider, request) @@ -61,7 +71,7 @@ class HttpCacheMiddleware: if self.ignore_missing: self.stats.inc_value('httpcache/ignore', spider=spider) raise IgnoreRequest("Ignored request not in cache: %s" % request) - return # first time request + return None # first time request # Return cached response only if not expired cachedresponse.flags.append('cached') @@ -73,7 +83,9 @@ class HttpCacheMiddleware: # process_response hook request.meta['cached_response'] = cachedresponse - def process_response(self, request, response, spider): + return None + + def process_response(self, request: Request, response: Response, spider: Spider) -> Response: if request.meta.get('dont_cache', False): return response @@ -85,7 +97,7 @@ class HttpCacheMiddleware: # RFC2616 requires origin server to set Date header, # https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 if 'Date' not in response.headers: - response.headers['Date'] = formatdate(usegmt=1) + response.headers['Date'] = formatdate(usegmt=True) # Do not validate first-hand responses cachedresponse = request.meta.pop('cached_response', None) @@ -102,13 +114,18 @@ class HttpCacheMiddleware: self._cache_response(spider, response, request, cachedresponse) return response - def process_exception(self, request, exception, spider): + def process_exception( + self, request: Request, exception: Exception, spider: Spider + ) -> Optional[Response]: cachedresponse = request.meta.pop('cached_response', None) if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS): self.stats.inc_value('httpcache/errorrecovery', spider=spider) return cachedresponse + return None - def _cache_response(self, spider, response, request, cachedresponse): + def _cache_response( + self, spider: Spider, response: Response, request: Request, cachedresponse: Optional[Response] + ) -> None: if self.policy.should_cache_response(response, request): self.stats.inc_value('httpcache/store', spider=spider) self.storage.store_response(spider, request, response) From e70975f0bb4e8378bf0e00ae7a7014247cd59441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 17 Aug 2020 15:10:08 +0200 Subject: [PATCH 245/568] Allow overwriting feeds (#4512) Co-authored-by: Yuval Hager --- docs/faq.rst | 6 +- docs/intro/overview.rst | 26 +- docs/intro/tutorial.rst | 13 +- docs/topics/feed-exports.rst | 47 +++- scrapy/commands/__init__.py | 15 +- scrapy/extensions/feedexport.py | 142 +++++++--- scrapy/utils/conf.py | 44 +++- scrapy/utils/ftp.py | 6 +- scrapy/utils/python.py | 3 +- tests/ftpserver.py | 24 ++ tests/mockserver.py | 26 ++ tests/requirements-py3.txt | 1 + tests/test_commands.py | 126 +++++++++ tests/test_feedexport.py | 442 ++++++++++++++++++++++++++++---- tests/test_utils_conf.py | 16 ++ tests/test_utils_python.py | 4 + 16 files changed, 791 insertions(+), 150 deletions(-) create mode 100644 tests/ftpserver.py diff --git a/docs/faq.rst b/docs/faq.rst index ea2c8216f..9346ec358 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -236,15 +236,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file? To dump into a JSON file:: - scrapy crawl myspider -o items.json + scrapy crawl myspider -O items.json To dump into a CSV file:: - scrapy crawl myspider -o items.csv + scrapy crawl myspider -O items.csv To dump into a XML file:: - scrapy crawl myspider -o items.xml + scrapy crawl myspider -O items.xml For more information see :ref:`topics-feed-exports` diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 01986b594..dd80c7bd0 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination:: if next_page is not None: yield response.follow(next_page, self.parse) - Put this in a text file, name it to something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: - scrapy runspider quotes_spider.py -o quotes.json + scrapy runspider quotes_spider.py -o quotes.jl +When this finishes you will have in the ``quotes.jl`` file a list of the +quotes in JSON Lines format, containing text and author, looking like this:: -When this finishes you will have in the ``quotes.json`` file a list of the -quotes in JSON format, containing text and author, looking like this (reformatted -here for better readability):: - - [{ - "author": "Jane Austen", - "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d" - }, - { - "author": "Groucho Marx", - "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d" - }, - { - "author": "Steve Martin", - "text": "\u201cA day without sunshine is like, you know, night.\u201d" - }, - ...] + {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} + {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"} + {"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"} + ... What just happened? diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 5f35dc936..f96c78887 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -464,16 +464,15 @@ Storing the scraped data The simplest way to store the scraped data is by using :ref:`Feed exports `, with the following command:: - scrapy crawl quotes -o quotes.json + scrapy crawl quotes -O quotes.json That will generate an ``quotes.json`` file containing all scraped items, serialized in `JSON`_. -For historic reasons, Scrapy appends to a given file instead of overwriting -its contents. If you run this command twice without removing the file -before the second time, you'll end up with a broken JSON file. - -You can also use other formats, like `JSON Lines`_:: +The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead +to append new content to any existing file. However, appending to a JSON file +makes the file contents invalid JSON. When appending to a file, consider +using a different serialization format, such as `JSON Lines`_:: scrapy crawl quotes -o quotes.jl @@ -704,7 +703,7 @@ Using spider arguments You can provide command line arguments to your spiders by using the ``-a`` option when running them:: - scrapy crawl quotes -o quotes-humor.json -a tag=humor + scrapy crawl quotes -O quotes-humor.json -a tag=humor These arguments are passed to the Spider's ``__init__`` method and become spider attributes by default. diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0f0f258dc..cd4f7cf29 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -291,6 +291,7 @@ Default: ``{}`` A dictionary in which every key is a feed URI (or a :class:`pathlib.Path` object) and each value is a nested dictionary containing configuration parameters for the specific feed. + This setting is required for enabling the feed export feature. See :ref:`topics-feed-storage-backends` for supported URI schemes. @@ -318,17 +319,43 @@ For instance:: } The following is a list of the accepted keys and the setting that is used -as a fallback value if that key is not provided for a specific feed definition. +as a fallback value if that key is not provided for a specific feed definition: + +- ``format``: the :ref:`serialization format `. + + This setting is mandatory, there is no fallback value. + +- ``batch_item_count``: falls back to + :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + +- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`. + +- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. + +- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. + +- ``overwrite``: whether to overwrite the file if it already exists + (``True``) or append to its content (``False``). + + The default value depends on the :ref:`storage backend + `: + + - :ref:`topics-feed-storage-fs`: ``False`` + + - :ref:`topics-feed-storage-ftp`: ``True`` + + .. note:: Some FTP servers may not support appending to files (the + ``APPE`` FTP command). + + - :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported + `_) + + - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) + +- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`. + +- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. -* ``format``: the serialization format to be used for the feed. - See :ref:`topics-feed-format` for possible values. - Mandatory, no fallback setting -* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` -* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING` -* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` -* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` -* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` -* ``uri_params``: falls back to :setting:`FEED_URI_PARAMS` .. setting:: FEED_EXPORT_ENCODING diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 57ce4e522..cfd940fe7 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand): parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE (use - for stdout)") + help="append scraped items to the end of FILE (use - for stdout)") + parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append", + help="dump scraped items into FILE, overwriting any existing file") parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") + help="format to use for dumping items") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -125,6 +127,11 @@ class BaseRunSpiderCommand(ScrapyCommand): opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) + if opts.output or opts.overwrite_output: + feeds = feed_process_params_from_cli( + self.settings, + opts.output, + opts.output_format, + opts.overwrite_output, + ) self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index b7a4e362e..980825499 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -24,17 +24,34 @@ from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.python import without_none_values +from scrapy.utils.python import get_func_args, without_none_values logger = logging.getLogger(__name__) +def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): + argument_names = get_func_args(builder) + if 'feed_options' in argument_names: + kwargs['feed_options'] = feed_options + else: + warnings.warn( + "{} does not support the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove this " + "warning. This parameter will become mandatory in a future " + "version of Scrapy." + .format(builder.__qualname__), + category=ScrapyDeprecationWarning + ) + return builder(*preargs, uri, *args, **kwargs) + + class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" - def __init__(uri): - """Initialize the storage with the parameters given in the URI""" + def __init__(uri, *, feed_options=None): + """Initialize the storage with the parameters given in the URI and the + feed-specific options (see :setting:`FEEDS`)""" def open(spider): """Open the storage for the given spider. It must return a file-like @@ -64,10 +81,15 @@ class BlockingFeedStorage: @implementer(IFeedStorage) class StdoutFeedStorage: - def __init__(self, uri, _stdout=None): + def __init__(self, uri, _stdout=None, *, feed_options=None): if not _stdout: _stdout = sys.stdout.buffer self._stdout = _stdout + if feed_options and feed_options.get('overwrite', False) is True: + logger.warning('Standard output (stdout) storage does not support ' + 'overwriting. To suppress this warning, remove the ' + 'overwrite option from your FEEDS setting, or set ' + 'it to False.') def open(self, spider): return self._stdout @@ -79,14 +101,16 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri): + def __init__(self, uri, *, feed_options=None): self.path = file_uri_to_path(uri) + feed_options = feed_options or {} + self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab' def open(self, spider): dirname = os.path.dirname(self.path) if dirname and not os.path.exists(dirname): os.makedirs(dirname) - return open(self.path, 'ab') + return open(self.path, self.write_mode) def store(self, file): file.close() @@ -94,7 +118,8 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None, acl=None): + def __init__(self, uri, access_key=None, secret_key=None, acl=None, *, + feed_options=None): u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key @@ -111,14 +136,20 @@ class S3FeedStorage(BlockingFeedStorage): else: import boto self.connect_s3 = boto.connect_s3 + if feed_options and feed_options.get('overwrite', True) is False: + logger.warning('S3 does not support appending to files. To ' + 'suppress this warning, remove the overwrite ' + 'option from your FEEDS setting or set it to True.') @classmethod - def from_crawler(cls, crawler, uri): - return cls( - uri=uri, + def from_crawler(cls, crawler, uri, *, feed_options=None): + return build_storage( + cls, + uri, access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], - acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None + acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, + feed_options=feed_options, ) def _store_in_thread(self, file): @@ -135,6 +166,7 @@ class S3FeedStorage(BlockingFeedStorage): kwargs = {'policy': self.acl} if self.acl else {} key.set_contents_from_file(file, **kwargs) key.close() + file.close() class GCSFeedStorage(BlockingFeedStorage): @@ -165,27 +197,31 @@ class GCSFeedStorage(BlockingFeedStorage): class FTPFeedStorage(BlockingFeedStorage): - def __init__(self, uri, use_active_mode=False): + def __init__(self, uri, use_active_mode=False, *, feed_options=None): u = urlparse(uri) self.host = u.hostname self.port = int(u.port or '21') self.username = u.username - self.password = unquote(u.password) + self.password = unquote(u.password or '') self.path = u.path self.use_active_mode = use_active_mode + self.overwrite = not feed_options or feed_options.get('overwrite', True) @classmethod - def from_crawler(cls, crawler, uri): - return cls( - uri=uri, - use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE') + def from_crawler(cls, crawler, uri, *, feed_options=None): + return build_storage( + cls, + uri, + crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'), + feed_options=feed_options, ) def _store_in_thread(self, file): ftp_store_file( path=self.path, file=file, host=self.host, port=self.port, username=self.username, - password=self.password, use_active_mode=self.use_active_mode + password=self.password, use_active_mode=self.use_active_mode, + overwrite=self.overwrite, ) @@ -242,32 +278,32 @@ class FeedExporter: category=ScrapyDeprecationWarning, stacklevel=2, ) uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects - feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} - self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings) + feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} + self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' - for uri, feed in self.settings.getdict('FEEDS').items(): + for uri, feed_options in self.settings.getdict('FEEDS').items(): uri = str(uri) # handle pathlib.Path objects - self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings) + self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - for uri, feed in self.feeds.items(): - if not self._storage_supported(uri): + for uri, feed_options in self.feeds.items(): + if not self._storage_supported(uri, feed_options): raise NotConfigured if not self._settings_are_valid(): raise NotConfigured - if not self._exporter_supported(feed['format']): + if not self._exporter_supported(feed_options['format']): raise NotConfigured def open_spider(self, spider): - for uri, feed in self.feeds.items(): - uri_params = self._get_uri_params(spider, feed['uri_params']) + for uri, feed_options in self.feeds.items(): + uri_params = self._get_uri_params(spider, feed_options['uri_params']) self.slots.append(self._start_new_batch( batch_id=1, uri=uri % uri_params, - feed=feed, + feed_options=feed_options, spider=spider, uri_template=uri, )) @@ -306,32 +342,32 @@ class FeedExporter: ) return d - def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): + def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): """ Redirect the output data stream to a new file. Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified :param batch_id: sequence number of current batch :param uri: uri of the new batch to start - :param feed: dict with parameters of feed + :param feed_options: dict with parameters of feed :param spider: user spider :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ - storage = self._get_storage(uri) + storage = self._get_storage(uri, feed_options) file = storage.open(spider) exporter = self._get_exporter( file=file, - format=feed['format'], - fields_to_export=feed['fields'], - encoding=feed['encoding'], - indent=feed['indent'], + format=feed_options['format'], + fields_to_export=feed_options['fields'], + encoding=feed_options['encoding'], + indent=feed_options['indent'], ) slot = _FeedSlot( file=file, exporter=exporter, storage=storage, uri=uri, - format=feed['format'], - store_empty=feed['store_empty'], + format=feed_options['format'], + store_empty=feed_options['store_empty'], batch_id=batch_id, uri_template=uri_template, ) @@ -355,7 +391,7 @@ class FeedExporter: slots.append(self._start_new_batch( batch_id=slot.batch_id + 1, uri=slot.uri_template % uri_params, - feed=self.feeds[slot.uri_template], + feed_options=self.feeds[slot.uri_template], spider=spider, uri_template=slot.uri_template, )) @@ -394,11 +430,11 @@ class FeedExporter: return False return True - def _storage_supported(self, uri): + def _storage_supported(self, uri, feed_options): scheme = urlparse(uri).scheme if scheme in self.storages: try: - self._get_storage(uri) + self._get_storage(uri, feed_options) return True except NotConfigured as e: logger.error("Disabled feed storage scheme: %(scheme)s. " @@ -416,8 +452,30 @@ class FeedExporter: def _get_exporter(self, file, format, *args, **kwargs): return self._get_instance(self.exporters[format], file, *args, **kwargs) - def _get_storage(self, uri): - return self._get_instance(self.storages[urlparse(uri).scheme], uri) + def _get_storage(self, uri, feed_options): + """Fork of create_instance specific to feed storage classes + + It supports not passing the *feed_options* parameters to classes that + do not support it, and issuing a deprecation warning instead. + """ + feedcls = self.storages[urlparse(uri).scheme] + crawler = getattr(self, 'crawler', None) + + def build_instance(builder, *preargs): + return build_storage(builder, uri, preargs=preargs) + + if crawler and hasattr(feedcls, 'from_crawler'): + instance = build_instance(feedcls.from_crawler, crawler) + method_name = 'from_crawler' + elif hasattr(feedcls, 'from_settings'): + instance = build_instance(feedcls.from_settings, self.settings) + method_name = 'from_settings' + else: + instance = build_instance(feedcls) + method_name = '__new__' + if instance is None: + raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name)) + return instance def _get_uri_params(self, spider, uri_params, slot=None): params = {} diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index a83076c47..90a52b25b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -127,7 +127,8 @@ def feed_complete_default_values_from_settings(feed, settings): return out -def feed_process_params_from_cli(settings, output, output_format=None): +def feed_process_params_from_cli(settings, output, output_format=None, + overwrite_output=None): """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary @@ -139,22 +140,39 @@ def feed_process_params_from_cli(settings, output, output_format=None): def check_valid_format(output_format): if output_format not in valid_output_formats: - raise UsageError("Unrecognized output format '%s', set one after a" - " colon using the -o option (i.e. -o :)" - " or as a file extension, from the supported list %s" % - (output_format, tuple(valid_output_formats))) + raise UsageError( + "Unrecognized output format '%s'. Set a supported one (%s) " + "after a colon at the end of the output URI (i.e. -o/-O " + ":) or as a file extension." % ( + output_format, + tuple(valid_output_formats), + ) + ) + + overwrite = False + if overwrite_output: + if output: + raise UsageError( + "Please use only one of -o/--output and -O/--overwrite-output" + ) + output = overwrite_output + overwrite = True if output_format: if len(output) == 1: check_valid_format(output_format) - warnings.warn('The -t command line option is deprecated in favor' - ' of specifying the output format within the -o' - ' option, please check the -o option docs for more details', - category=ScrapyDeprecationWarning, stacklevel=2) + message = ( + 'The -t command line option is deprecated in favor of ' + 'specifying the output format within the output URI. See the ' + 'documentation of the -o and -O options for more information.', + ) + warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} else: - raise UsageError('The -t command line option cannot be used if multiple' - ' output files are specified with the -o option') + raise UsageError( + 'The -t command-line option cannot be used if multiple output ' + 'URIs are specified' + ) result = {} for element in output: @@ -168,8 +186,10 @@ def feed_process_params_from_cli(settings, output, output_format=None): feed_uri = 'stdout:' check_valid_format(feed_format) result[feed_uri] = {'format': feed_format} + if overwrite: + result[feed_uri]['overwrite'] = True - # FEEDS setting should take precedence over the -o and -t CLI options + # FEEDS setting should take precedence over the matching CLI options result.update(settings.getdict('FEEDS')) return result diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index f07bdd748..19d56d6ec 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -20,7 +20,7 @@ def ftp_makedirs_cwd(ftp, path, first_call=True): def ftp_store_file( *, path, file, host, port, - username, password, use_active_mode=False): + username, password, use_active_mode=False, overwrite=True): """Opens a FTP connection with passed credentials,sets current directory to the directory extracted from given path, then uploads the file to server """ @@ -32,4 +32,6 @@ def ftp_store_file( file.seek(0) dirname, filename = posixpath.split(path) ftp_makedirs_cwd(ftp, dirname) - ftp.storbinary('STOR %s' % filename, file) + command = 'STOR' if overwrite else 'APPE' + ftp.storbinary('%s %s' % (command, filename), file) + file.close() diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 59f1b8371..1f2333264 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -198,7 +198,8 @@ def _getargspec_py23(func): def get_func_args(func, stripself=False): """Return the argument name list of a callable""" if inspect.isfunction(func): - func_args, _, _, _ = _getargspec_py23(func) + spec = inspect.getfullargspec(func) + func_args = spec.args + spec.kwonlyargs elif inspect.isclass(func): return get_func_args(func.__init__, True) elif inspect.ismethod(func): diff --git a/tests/ftpserver.py b/tests/ftpserver.py new file mode 100644 index 000000000..6f0289e08 --- /dev/null +++ b/tests/ftpserver.py @@ -0,0 +1,24 @@ +from argparse import ArgumentParser + +from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib.handlers import FTPHandler +from pyftpdlib.servers import FTPServer + + +def main(): + parser = ArgumentParser() + parser.add_argument('-d', '--directory') + args = parser.parse_args() + + authorizer = DummyAuthorizer() + full_permissions = 'elradfmwMT' + authorizer.add_anonymous(args.directory, perm=full_permissions) + handler = FTPHandler + handler.authorizer = authorizer + address = ('127.0.0.1', 2121) + server = FTPServer(address, handler) + server.serve_forever() + + +if __name__ == '__main__': + main() diff --git a/tests/mockserver.py b/tests/mockserver.py index 1f40473ba..48d7b8d37 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -3,7 +3,10 @@ import json import os import random import sys +from pathlib import Path +from shutil import rmtree from subprocess import Popen, PIPE +from tempfile import mkdtemp from urllib.parse import urlencode from OpenSSL import SSL @@ -256,6 +259,29 @@ class MockDNSServer: self.proc.communicate() +class MockFTPServer: + """Creates an FTP server on port 2121 with a default passwordless user + (anonymous) and a temporary root path that you can read from the + :attr:`path` attribute.""" + + def __enter__(self): + self.path = Path(mkdtemp()) + self.proc = Popen([sys.executable, '-u', '-m', 'tests.ftpserver', '-d', str(self.path)], + stderr=PIPE, env=get_testenv()) + for line in self.proc.stderr: + if b'starting FTP server' in line: + break + return self + + def __exit__(self, exc_type, exc_value, traceback): + rmtree(str(self.path)) + self.proc.kill() + self.proc.communicate() + + def url(self, path): + return 'ftp://127.0.0.1:2121/' + path + + def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( os.path.join(os.path.dirname(__file__), keyfile), diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index b51177abb..fe1cbc997 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -4,6 +4,7 @@ dataclasses; python_version == '3.6' mitmproxy; python_version >= '3.7' mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7' mitmproxy < 4; python_version < '3.6' +pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines diff --git a/tests/test_commands.py b/tests/test_commands.py index 109f006a8..f76f851e7 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -74,6 +74,7 @@ class ProjectTest(unittest.TestCase): def kill_proc(): p.kill() + p.communicate() assert False, 'Command took too much time to complete' timer = Timer(15, kill_proc) @@ -569,6 +570,55 @@ class BadSpider(scrapy.Spider): log = self.get_log(self.debug_log_spider, args=[]) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return [] +""" + args = ['-o', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log) + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return [] +""" + args = ['-O', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + return [] +""" + args = ['-o', 'example1.json', '-O', 'example2.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) + class BenchCommandTest(CommandTest): @@ -577,3 +627,79 @@ class BenchCommandTest(CommandTest): '-s', 'CLOSESPIDER_TIMEOUT=0.01') self.assertIn('INFO: Crawled', log) self.assertNotIn('Unhandled Error', log) + + +class CrawlCommandTest(CommandTest): + + def crawl(self, code, args=()): + fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) + with open(fname, 'w') as f: + f.write(code) + return self.proc('crawl', 'myspider', *args) + + def get_log(self, code, args=()): + _, _, stderr = self.crawl(code, args=args) + return stderr + + def test_no_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('It works!') + return [] +""" + log = self.get_log(spider_code) + self.assertIn("[myspider] DEBUG: It works!", log) + + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return [] +""" + args = ['-o', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log) + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return [] +""" + args = ['-O', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + return [] +""" + args = ['-o', 'example1.json', '-O', 'example2.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2afc25a7a..850485b5e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,6 +5,7 @@ import random import shutil import string import tempfile +import warnings from abc import ABC, abstractmethod from collections import defaultdict from io import BytesIO @@ -25,7 +26,7 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, @@ -46,7 +47,7 @@ from scrapy.utils.test import ( mock_google_cloud_storage, ) -from tests.mockserver import MockServer +from tests.mockserver import MockFTPServer, MockServer class FileFeedStorageTest(unittest.TestCase): @@ -75,8 +76,28 @@ class FileFeedStorageTest(unittest.TestCase): st = FileFeedStorage(path) verifyObject(IFeedStorage, st) + def _store(self, feed_options=None): + path = os.path.abspath(self.mktemp()) + storage = FileFeedStorage(path, feed_options=feed_options) + spider = scrapy.Spider("default") + file = storage.open(spider) + file.write(b"content") + storage.store(file) + return path + + def test_append(self): + path = self._store() + return self._assert_stores(FileFeedStorage(path), path, b"contentcontent") + + def test_overwrite(self): + path = self._store({"overwrite": True}) + return self._assert_stores( + FileFeedStorage(path, feed_options={"overwrite": True}), + path + ) + @defer.inlineCallbacks - def _assert_stores(self, storage, path): + def _assert_stores(self, storage, path, expected_content=b"content"): spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") @@ -84,7 +105,7 @@ class FileFeedStorageTest(unittest.TestCase): self.assertTrue(os.path.exists(path)) try: with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"content") + self.assertEqual(fp.read(), expected_content) finally: os.unlink(path) @@ -99,49 +120,74 @@ class FTPFeedStorageTest(unittest.TestCase): spider = TestSpider.from_crawler(crawler) return spider - def test_store(self): - uri = os.environ.get('FEEDTEST_FTP_URI') - path = os.environ.get('FEEDTEST_FTP_PATH') - if not (uri and path): - raise unittest.SkipTest("No FTP server available for testing") - st = FTPFeedStorage(uri) - verifyObject(IFeedStorage, st) - return self._assert_stores(st, path) + def _store(self, uri, content, feed_options=None, settings=None): + crawler = get_crawler(settings_dict=settings or {}) + storage = FTPFeedStorage.from_crawler( + crawler, + uri, + feed_options=feed_options, + ) + verifyObject(IFeedStorage, storage) + spider = self.get_test_spider() + file = storage.open(spider) + file.write(content) + return storage.store(file) - def test_store_active_mode(self): - uri = os.environ.get('FEEDTEST_FTP_URI') - path = os.environ.get('FEEDTEST_FTP_PATH') - if not (uri and path): - raise unittest.SkipTest("No FTP server available for testing") - use_active_mode = {'FEED_STORAGE_FTP_ACTIVE': True} - crawler = get_crawler(settings_dict=use_active_mode) - st = FTPFeedStorage.from_crawler(crawler, uri) - verifyObject(IFeedStorage, st) - return self._assert_stores(st, path) + def _assert_stored(self, path, content): + self.assertTrue(path.exists()) + try: + with path.open('rb') as fp: + self.assertEqual(fp.read(), content) + finally: + os.unlink(str(path)) + + @defer.inlineCallbacks + def test_append(self): + with MockFTPServer() as ftp_server: + filename = 'file' + url = ftp_server.url(filename) + feed_options = {'overwrite': False} + yield self._store(url, b"foo", feed_options=feed_options) + yield self._store(url, b"bar", feed_options=feed_options) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @defer.inlineCallbacks + def test_overwrite(self): + with MockFTPServer() as ftp_server: + filename = 'file' + url = ftp_server.url(filename) + yield self._store(url, b"foo") + yield self._store(url, b"bar") + self._assert_stored(ftp_server.path / filename, b"bar") + + @defer.inlineCallbacks + def test_append_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {'FEED_STORAGE_FTP_ACTIVE': True} + filename = 'file' + url = ftp_server.url(filename) + feed_options = {'overwrite': False} + yield self._store(url, b"foo", feed_options=feed_options, settings=settings) + yield self._store(url, b"bar", feed_options=feed_options, settings=settings) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @defer.inlineCallbacks + def test_overwrite_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {'FEED_STORAGE_FTP_ACTIVE': True} + filename = 'file' + url = ftp_server.url(filename) + yield self._store(url, b"foo", settings=settings) + yield self._store(url, b"bar", settings=settings) + self._assert_stored(ftp_server.path / filename, b"bar") def test_uri_auth_quote(self): # RFC3986: 3.2.1. User Information pw_quoted = quote(string.punctuation, safe='') - st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted) + st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted, + {}) self.assertEqual(st.password, string.punctuation) - @defer.inlineCallbacks - def _assert_stores(self, storage, path): - spider = self.get_test_spider() - file = storage.open(spider) - file.write(b"content") - yield storage.store(file) - self.assertTrue(os.path.exists(path)) - try: - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"content") - # again, to check s3 objects are overwritten - yield storage.store(BytesIO(b"new content")) - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"new content") - finally: - os.unlink(path) - class BlockingFeedStorageTest(unittest.TestCase): @@ -190,8 +236,10 @@ class S3FeedStorageTest(unittest.TestCase): 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} crawler = get_crawler(settings_dict=aws_credentials) # Instantiate with crawler - storage = S3FeedStorage.from_crawler(crawler, - 's3://mybucket/export.csv') + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv', + ) self.assertEqual(storage.access_key, 'settings_key') self.assertEqual(storage.secret_key, 'settings_secret') # Instantiate directly @@ -254,7 +302,7 @@ class S3FeedStorageTest(unittest.TestCase): crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler( crawler, - 's3://mybucket/export.csv' + 's3://mybucket/export.csv', ) self.assertEqual(storage.access_key, 'access_key') self.assertEqual(storage.secret_key, 'secret_key') @@ -269,7 +317,7 @@ class S3FeedStorageTest(unittest.TestCase): crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler( crawler, - 's3://mybucket/export.csv' + 's3://mybucket/export.csv', ) self.assertEqual(storage.access_key, 'access_key') self.assertEqual(storage.secret_key, 'secret_key') @@ -370,6 +418,27 @@ class S3FeedStorageTest(unittest.TestCase): key.set_contents_from_file.call_args ) + def test_overwrite_default(self): + with LogCapture() as log: + S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertNotIn('S3 does not support appending to files', str(log)) + + def test_overwrite_false(self): + with LogCapture() as log: + S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl', + feed_options={'overwrite': False}, + ) + self.assertIn('S3 does not support appending to files', str(log)) + class GCSFeedStorageTest(unittest.TestCase): @@ -439,12 +508,22 @@ class StdoutFeedStorageTest(unittest.TestCase): yield storage.store(file) self.assertEqual(out.getvalue(), b"content") + def test_overwrite_default(self): + with LogCapture() as log: + StdoutFeedStorage('stdout:') + self.assertNotIn('Standard output (stdout) storage does not support overwriting', str(log)) + + def test_overwrite_true(self): + with LogCapture() as log: + StdoutFeedStorage('stdout:', feed_options={'overwrite': True}) + self.assertIn('Standard output (stdout) storage does not support overwriting', str(log)) + class FromCrawlerMixin: init_with_crawler = False @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler, *args, feed_options=None, **kwargs): cls.init_with_crawler = True return cls(*args, **kwargs) @@ -454,7 +533,11 @@ class FromCrawlerCsvItemExporter(CsvItemExporter, FromCrawlerMixin): class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): - pass + + @classmethod + def from_crawler(cls, crawler, *args, feed_options=None, **kwargs): + cls.init_with_crawler = True + return cls(*args, feed_options=feed_options, **kwargs) class DummyBlockingFeedStorage(BlockingFeedStorage): @@ -588,8 +671,8 @@ class FeedExportTest(FeedExportTestBase): FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { - printf_escape(path_to_url(file_path)): feed - for file_path, feed in FEEDS.items() + printf_escape(path_to_url(file_path)): feed_options + for file_path, feed_options in FEEDS.items() } content = {} @@ -599,12 +682,12 @@ class FeedExportTest(FeedExportTestBase): spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) - for file_path, feed in FEEDS.items(): + for file_path, feed_options in FEEDS.items(): if not os.path.exists(str(file_path)): continue with open(str(file_path), 'rb') as f: - content[feed['format']] = f.read() + content[feed_options['format']] = f.read() finally: for file_path in FEEDS.keys(): @@ -1542,3 +1625,262 @@ class BatchDeliveriesTest(FeedExportTestBase): content = json.loads(content.decode('utf-8')) expected_batch, items = items[:batch_size], items[batch_size:] self.assertEqual(expected_batch, content) + + +class FeedExportInitTest(unittest.TestCase): + + def test_unsupported_storage(self): + settings = { + 'FEEDS': { + 'unsupported://uri': {}, + }, + } + crawler = get_crawler(settings_dict=settings) + with self.assertRaises(NotConfigured): + FeedExporter.from_crawler(crawler) + + def test_unsupported_format(self): + settings = { + 'FEEDS': { + 'file://path': { + 'format': 'unsupported_format', + }, + }, + } + crawler = get_crawler(settings_dict=settings) + with self.assertRaises(NotConfigured): + FeedExporter.from_crawler(crawler) + + +class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage): + + def __init__(self, uri): + super().__init__(uri) + + +class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.StdoutFeedStorageWithoutFeedOptions' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "StdoutFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + +class FileFeedStorageWithoutFeedOptions(FileFeedStorage): + + def __init__(self, uri): + super().__init__(uri) + + +class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + maxDiff = None + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.FileFeedStorageWithoutFeedOptions' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "FileFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + +class S3FeedStorageWithoutFeedOptions(S3FeedStorage): + + def __init__(self, uri, access_key, secret_key, acl): + super().__init__(uri, access_key, secret_key, acl) + + +class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): + + @classmethod + def from_crawler(cls, crawler, uri): + return super().from_crawler(crawler, uri) + + +class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + maxDiff = None + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptions' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "S3FeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + def test_from_crawler(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptionsWithFromCrawler' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler " + "does not support the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + +class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage): + + def __init__(self, uri, use_active_mode=False): + super().__init__(uri) + + +class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage): + + @classmethod + def from_crawler(cls, crawler, uri): + return super().from_crawler(crawler, uri) + + +class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + maxDiff = None + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptions' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "FTPFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + def test_from_crawler(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptionsWithFromCrawler' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler " + "does not support the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index f3ef36127..ccc65c4fd 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -141,6 +141,22 @@ class FeedExportConfigTestCase(unittest.TestCase): feed_process_params_from_cli(settings, ['-:pickle']) ) + def test_feed_export_config_overwrite(self): + settings = Settings() + self.assertEqual( + {'output.json': {'format': 'json', 'overwrite': True}}, + feed_process_params_from_cli(settings, [], None, ['output.json']) + ) + + def test_output_and_overwrite_output(self): + with self.assertRaises(UsageError): + feed_process_params_from_cli( + Settings(), + ['output1.json'], + None, + ['output2.json'], + ) + def test_feed_complete_default_values_from_settings_empty(self): feed = {} settings = Settings({ diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3f93f509e..c298d0bd2 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -179,6 +179,9 @@ class UtilsPythonTestCase(unittest.TestCase): def f2(a, b=None, c=None): pass + def f3(a, b=None, *, c=None): + pass + class A: def __init__(self, a, b, c): pass @@ -199,6 +202,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(f1), ['a', 'b', 'c']) self.assertEqual(get_func_args(f2), ['a', 'b', 'c']) + self.assertEqual(get_func_args(f3), ['a', 'b', 'c']) self.assertEqual(get_func_args(A), ['a', 'b', 'c']) self.assertEqual(get_func_args(a.method), ['a', 'b', 'c']) self.assertEqual(get_func_args(partial_f1), ['b', 'c']) From d9e69bfb51d5ed5a08d57878b29a6e7db8ef9d15 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Aug 2020 19:46:24 +0500 Subject: [PATCH 246/568] Re-enable TLS 1.2 in cipher tests. --- tests/mockserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 48d7b8d37..6f0c274b9 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -289,8 +289,8 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c ) if cipher_string: ctx = factory.getContext() - # disabling TLS1.2+ because it unconditionally enables some strong ciphers - ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3) + # disabling TLS1.3 because it unconditionally enables some strong ciphers + ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3) ctx.set_cipher_list(to_bytes(cipher_string)) return factory From a87ab71d1061585e41864d7283557bbe9823a91b Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 18 Aug 2020 04:47:09 +0530 Subject: [PATCH 247/568] refactor(http2): metadata for Stream - Add Note about HTTP/2 Cleartext not supported in settings.rst --- docs/topics/settings.rst | 7 +++ scrapy/core/http2/protocol.py | 2 +- scrapy/core/http2/stream.py | 73 +++++++++++++++---------------- scrapy/core/http2/types.py | 23 ++++++++++ setup.py | 2 +- tests/test_downloader_handlers.py | 2 +- 6 files changed, 68 insertions(+), 41 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d4d4f9332..0dad30b29 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -627,6 +627,13 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', } +.. note:: + + Scrapy currently does not support HTTP/2 Cleartext (h2c) since none + of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_). + +.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index fee391af6..7a3156541 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -289,7 +289,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._conn_lost_deferred.callback(self._conn_lost_errors) for stream in self.streams.values(): - if stream.request_sent: + if stream.metadata['request_sent']: close_reason = StreamCloseReason.CONNECTION_LOST else: close_reason = StreamCloseReason.INACTIVE diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 1e136fbd5..bddf50a56 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -5,14 +5,14 @@ from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes -from h2.exceptions import H2Error, StreamClosedError +from h2.exceptions import H2Error, StreamClosedError, ProtocolError from hpack import HeaderTuple from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from scrapy.core.http2.types import H2ResponseDict +from scrapy.core.http2.types import H2ResponseDict, H2StreamMetadataDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes @@ -100,24 +100,14 @@ class Stream: self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) - self.request_start_time = None - - self.content_length = 0 if self._request.body is None else len(self._request.body) - - # Flag to keep track whether this stream has initiated the request - self.request_sent = False - - # Flag to track whether we have logged about exceeding download warnsize - self._reached_warnsize = False - - # Each time we send a data frame, we will decrease value by the amount send. - self.remaining_content_length = self.content_length - - # Flag to keep track whether we have closed this stream - self.stream_closed_local = False - - # Flag to keep track whether the server has closed the stream - self.stream_closed_server = False + self.metadata: H2StreamMetadataDict = { + 'request_content_length': 0 if self._request.body is None else len(self._request.body), + 'request_sent': False, + 'reached_warnsize': False, + 'remaining_content_length': 0 if self._request.body is None else len(self._request.body), + 'stream_closed_local': False, + 'stream_closed_server': False, + } # Private variable used to build the response # this response is then converted to appropriate Response class @@ -132,7 +122,7 @@ class Stream: # Close this stream as gracefully as possible # If the associated request is initiated we reset this stream # else we directly call close() method - if self.request_sent: + if self.metadata['request_sent']: self.reset_stream(StreamCloseReason.CANCELLED) else: self.close(StreamCloseReason.CANCELLED) @@ -160,7 +150,7 @@ class Stream: self._response['flow_controlled_size'] > self._download_warnsize or content_length_header > self._download_warnsize ) - and not self._reached_warnsize + and not self.metadata['reached_warnsize'] ) def get_response(self) -> Deferred: @@ -220,7 +210,7 @@ class Stream: if self.check_request_url(): headers = self._get_request_headers() self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False) - self.request_sent = True + self.metadata['request_sent'] = True self.send_data() else: # Close this stream calling the response errback @@ -238,7 +228,7 @@ class Stream: and has initiated request already by sending HEADER frame. If not then stream will raise ProtocolError (raise by h2 state machine). """ - if self.stream_closed_local: + if self.metadata['stream_closed_local']: raise StreamClosedError(self.stream_id) # Firstly, check what the flow control window is for current stream. @@ -249,24 +239,24 @@ class Stream: # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. - bytes_to_send_size = min(window_size, self.remaining_content_length) + bytes_to_send_size = min(window_size, self.metadata['remaining_content_length']) # We now need to send a number of data frames. while bytes_to_send_size > 0: chunk_size = min(bytes_to_send_size, max_frame_size) - data_chunk_start_id = self.content_length - self.remaining_content_length + data_chunk_start_id = self.metadata['request_content_length'] - self.metadata['remaining_content_length'] data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size] self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False) bytes_to_send_size = bytes_to_send_size - chunk_size - self.remaining_content_length = self.remaining_content_length - chunk_size + self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size - self.remaining_content_length = max(0, self.remaining_content_length) + self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length']) # End the stream if no more data needs to be send - if self.remaining_content_length == 0: + if self.metadata['remaining_content_length'] == 0: self._protocol.conn.end_stream(self.stream_id) # Q. What about the rest of the data? @@ -277,7 +267,11 @@ class Stream: Send data that earlier could not be sent as we were blocked behind the flow control. """ - if self.remaining_content_length and not self.stream_closed_server and self.request_sent: + if ( + self.metadata['remaining_content_length'] + and not self.metadata['stream_closed_server'] + and self.metadata['request_sent'] + ): self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int) -> None: @@ -290,7 +284,7 @@ class Stream: return if self._log_warnsize: - self._reached_warnsize = True + self.metadata['reached_warnsize'] = True warning_msg = ( f'Received more ({self._response["flow_controlled_size"]}) bytes than download ' f'warn size ({self._download_warnsize}) in request {self._request}' @@ -314,7 +308,7 @@ class Stream: return if self._log_warnsize: - self._reached_warnsize = True + self.metadata['reached_warnsize'] = True warning_msg = ( f'Expected response size ({expected_size}) larger than ' f'download warn size ({self._download_warnsize}) in request {self._request}' @@ -323,18 +317,18 @@ class Stream: def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None: """Close this stream by sending a RST_FRAME to the remote peer""" - if self.stream_closed_local: + if self.metadata['stream_closed_local']: raise StreamClosedError(self.stream_id) # Clear buffer earlier to avoid keeping data in memory for a long time self._response['body'].truncate(0) - self.stream_closed_local = True + self.metadata['stream_closed_local'] = True self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) self.close(reason) def _is_data_lost(self) -> bool: - assert self.stream_closed_server + assert self.metadata['stream_closed_server'] expected_size = self._response['flow_controlled_size'] received_body_size = int(self._response['headers'][b'Content-Length']) @@ -349,7 +343,7 @@ class Stream: ) -> None: """Based on the reason sent we will handle each case. """ - if self.stream_closed_server: + if self.metadata['stream_closed_server']: raise StreamClosedError(self.stream_id) if not isinstance(reason, StreamCloseReason): @@ -362,7 +356,7 @@ class Stream: if not from_protocol: self._protocol.pop_stream(self.stream_id) - self.stream_closed_server = True + self.metadata['stream_closed_server'] = True # We do not check for Content-Length or Transfer-Encoding in response headers # and add `partial` flag as in HTTP/1.1 as 'A request or response that includes @@ -402,7 +396,10 @@ class Stream: elif reason is StreamCloseReason.RESET: self._deferred_response.errback(ResponseFailed([ - Failure(f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM') + Failure( + f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM', + ProtocolError + ) ])) elif reason is StreamCloseReason.CONNECTION_LOST: diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index d2aa1a9d8..ff8d94066 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -31,6 +31,29 @@ class H2ConnectionMetadataDict(TypedDict): default_download_warnsize: int +class H2StreamMetadataDict(TypedDict): + """Metadata of an HTTP/2 connection stream + initialized when stream is instantiated + """ + + request_content_length: int + + # Flag to keep track whether the stream has initiated the request + request_sent: bool + + # Flag to track whether we have logged about exceeding download warnsize + reached_warnsize: bool + + # Each time we send a data frame, we will decrease value by the amount send. + remaining_content_length: int + + # Flag to keep track whether we have closed this stream + stream_closed_local: bool + + # Flag to keep track whether the server has closed the stream + stream_closed_server: bool + + class H2ResponseDict(TypedDict): # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. diff --git a/setup.py b/setup.py index c8733ae96..66f369a71 100644 --- a/setup.py +++ b/setup.py @@ -97,4 +97,4 @@ setup( python_requires='>=3.5.2', install_requires=install_requires, extras_require=extras_require, -) \ No newline at end of file +) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2b3fa2aca..e3777ee1d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -232,7 +232,7 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) - self.host = u'localhost' + self.host = 'localhost' if self.scheme == 'https': # Using WrappingFactory do not enable HTTP/2 failing all the # tests with H2DownloadHandler From a206ac5f6f4e5eb057ecc535556a15087b250d40 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 18 Aug 2020 07:36:00 +0530 Subject: [PATCH 248/568] tests: disable python 3.5 for travis and azure --- .travis.yml | 25 +++++++++++++------------ azure-pipelines.yml | 7 ++++--- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0b55cda19..6628e8e43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,18 +18,19 @@ matrix: python: 3.7 # Keep in sync with .readthedocs.yml - env: TOXENV=typing python: 3.8 - - - env: TOXENV=pinned - python: 3.5.2 - - env: TOXENV=asyncio-pinned - python: 3.5.2 # We use additional code to support 3.5.3 and earlier - - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0 - - - env: TOXENV=py - python: 3.5 - - env: TOXENV=asyncio - python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 - - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 + +# ToDo: Remove once merged into master +# - env: TOXENV=pinned +# python: 3.5.2 +# - env: TOXENV=asyncio-pinned +# python: 3.5.2 # We use additional code to support 3.5.3 and earlier +# - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0 +# +# - env: TOXENV=py +# python: 3.5 +# - env: TOXENV=asyncio +# python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 +# - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 - env: TOXENV=py python: 3.6 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 710e42090..c77c128b3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -4,9 +4,10 @@ pool: vmImage: 'windows-latest' strategy: matrix: - Python35: - python.version: '3.5' - TOXENV: windows-pinned +# ToDo: Remove once merged into master +# Python35: +# python.version: '3.5' +# TOXENV: windows-pinned Python36: python.version: '3.6' Python37: From e3233b79deee6ad283ed4316135e30ab774c1078 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 19 Aug 2020 05:10:19 +0530 Subject: [PATCH 249/568] refactor(h2-stream): alphabetical order of imports --- scrapy/core/downloader/handlers/http2.py | 2 +- scrapy/core/http2/stream.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index f2ed40f9b..ddd813cec 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -29,7 +29,7 @@ class H2DownloadHandler: def from_crawler(cls, crawler): return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index bddf50a56..33302421e 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -5,7 +5,7 @@ from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes -from h2.exceptions import H2Error, StreamClosedError, ProtocolError +from h2.exceptions import H2Error, ProtocolError, StreamClosedError from hpack import HeaderTuple from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed From 30eb005639b77ea94e8859e1d041dfe53048cf77 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 19 Aug 2020 06:25:04 +0530 Subject: [PATCH 250/568] fix: InvalidNegotiatedProtocol __str__ method --- scrapy/core/http2/agent.py | 2 +- scrapy/core/http2/protocol.py | 2 +- scrapy/core/http2/stream.py | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index f4ac29bc6..f829cc5f8 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -68,7 +68,7 @@ class H2ConnectionPool: # Now as we have established a proper HTTP/2 connection # we fire all the deferred's with the connection instance - pending_requests = self._pending_requests.pop(key) + pending_requests = self._pending_requests.pop(key, None) while pending_requests: d = pending_requests.popleft() d.callback(conn) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 7a3156541..c6f00423a 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -39,7 +39,7 @@ class InvalidNegotiatedProtocol(H2Error): self.negotiated_protocol = negotiated_protocol def __str__(self) -> str: - return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}' + return f'InvalidNegotiatedProtocol: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}' class RemoteTerminatedConnection(H2Error): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 33302421e..df7470e11 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -31,6 +31,9 @@ class InactiveStreamClosed(ConnectionClosed): def __init__(self, request: Request): self.request = request + def __str__(self) -> str: + return f'InactiveStreamClosed: Connection was closed without sending the request {self.request!r}' + class InvalidHostname(H2Error): From 2f00666d749b4a736536bafc786e63be06113676 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 19 Aug 2020 07:31:52 +0530 Subject: [PATCH 251/568] refactor: move agents & context-factory --- scrapy/core/downloader/contextfactory.py | 22 ++++++++++++- scrapy/core/downloader/handlers/http2.py | 31 ++--------------- scrapy/core/http2/agent.py | 42 ++++++++++++++---------- 3 files changed, 49 insertions(+), 46 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 5768d8f8e..073ef16bf 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,10 +1,12 @@ import warnings from OpenSSL import SSL +from twisted.internet._sslverify import _setAcceptableProtocols from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer +from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions from scrapy.utils.misc import create_instance, load_object @@ -84,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): The default OpenSSL method is ``TLS_METHOD`` (also called ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ - def creatorForNetloc(self, hostname, port): + def creatorForNetloc(self, hostname, port): # trustRoot set to platformTrust() will use the platform's root CAs. # # This means that a website like https://www.cacert.org will be rejected @@ -97,6 +99,24 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): ) +@implementer(IPolicyForHTTPS) +class AcceptableProtocolsContextFactory: + """Context factory to used to override the acceptable protocols + to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN + negotiation. + """ + + def __init__(self, context_factory, acceptable_protocols): + verifyObject(IPolicyForHTTPS, context_factory) + self._wrapped_context_factory = context_factory + self._acceptable_protocols = acceptable_protocols + + def creatorForNetloc(self, hostname, port): + options = self._wrapped_context_factory.creatorForNetloc(hostname, port) + _setAcceptableProtocols(options._ctx, self._acceptable_protocols) + return options + + def load_context_factory_from_settings(settings, crawler): ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index ddd813cec..4be888bda 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,16 +1,15 @@ import warnings from time import time -from typing import Optional, Tuple +from typing import Optional from urllib.parse import urldefrag -from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.web.client import BrowserLikePolicyForHTTPS, URI +from twisted.web.client import URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse -from scrapy.core.http2.agent import H2Agent, H2ConnectionPool +from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider @@ -41,30 +40,6 @@ class H2DownloadHandler: self._pool.close_connections() -class ScrapyProxyH2Agent(H2Agent): - def __init__( - self, reactor: ReactorBase, - proxy_uri: URI, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None - ) -> None: - super(ScrapyProxyH2Agent, self).__init__( - reactor=reactor, - pool=pool, - context_factory=context_factory, - connect_timeout=connect_timeout, - bind_address=bind_address - ) - self._proxy_uri = proxy_uri - - def get_endpoint(self, uri: URI): - return self.endpoint_factory.endpointForURI(self._proxy_uri) - - def get_key(self, uri: URI) -> Tuple: - """We use the proxy uri instead of uri obtained from request url""" - return "http-proxy", self._proxy_uri.host, self._proxy_uri.port - - class ScrapyH2Agent: _Agent = H2Agent _ProxyAgent = ScrapyProxyH2Agent diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index f829cc5f8..d950c6cfb 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -2,17 +2,14 @@ from collections import deque from typing import Deque, Dict, List, Tuple, Optional from twisted.internet import defer -from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint from twisted.python.failure import Failure from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory from twisted.web.error import SchemeNotSupported -from twisted.web.iweb import IPolicyForHTTPS -from zope.interface import implementer -from zope.interface.verify import verifyObject +from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory from scrapy.http.request import Request from scrapy.settings import Settings @@ -96,19 +93,6 @@ class H2ConnectionPool: conn.transport.abortConnection() -@implementer(IPolicyForHTTPS) -class AcceptableProtocolsContextFactory: - def __init__(self, context_factory, acceptable_protocols: List[bytes]) -> None: - verifyObject(IPolicyForHTTPS, context_factory) - self._wrapped_context_factory = context_factory - self._acceptable_protocols = acceptable_protocols - - def creatorForNetloc(self, hostname, port) -> ClientTLSOptions: - options = self._wrapped_context_factory.creatorForNetloc(hostname, port) - _setAcceptableProtocols(options._ctx, self._acceptable_protocols) - return options - - class H2Agent: def __init__( self, reactor: ReactorBase, pool: H2ConnectionPool, @@ -144,3 +128,27 @@ class H2Agent: d = self._pool.get_connection(key, uri, endpoint) d.addCallback(lambda conn: conn.request(request, spider)) return d + + +class ScrapyProxyH2Agent(H2Agent): + def __init__( + self, reactor: ReactorBase, + proxy_uri: URI, pool: H2ConnectionPool, + context_factory=BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + ) -> None: + super(ScrapyProxyH2Agent, self).__init__( + reactor=reactor, + pool=pool, + context_factory=context_factory, + connect_timeout=connect_timeout, + bind_address=bind_address + ) + self._proxy_uri = proxy_uri + + def get_endpoint(self, uri: URI): + return self.endpoint_factory.endpointForURI(self._proxy_uri) + + def get_key(self, uri: URI) -> Tuple: + """We use the proxy uri instead of uri obtained from request url""" + return "http-proxy", self._proxy_uri.host, self._proxy_uri.port From 42383cc267393c3c5fb89c9108759125939ab3e5 Mon Sep 17 00:00:00 2001 From: sakshamb2113 <44064539+sakshamb2113@users.noreply.github.com> Date: Wed, 19 Aug 2020 12:48:14 +0530 Subject: [PATCH 252/568] Add a setting to customize the asyncio event loop (#4414) --- docs/topics/asyncio.rst | 12 +++++++++++ docs/topics/settings.rst | 20 ++++++++++++++++++ scrapy/crawler.py | 2 +- scrapy/settings/default_settings.py | 2 ++ scrapy/utils/log.py | 7 +++++++ scrapy/utils/reactor.py | 12 ++++++++--- tests/CrawlerProcess/asyncio_custom_loop.py | 17 +++++++++++++++ tests/requirements-py3.txt | 1 + tests/test_commands.py | 23 +++++++++++++++++++++ tests/test_crawler.py | 8 +++++++ 10 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 tests/CrawlerProcess/asyncio_custom_loop.py diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 038a459fd..bfb430d52 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -26,3 +26,15 @@ reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`:: install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') + +.. _using-custom-loops: + +Using custom asyncio loops +========================== + +You can also use custom asyncio event loops with the asyncio reactor. Set the +:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to +use it instead of the default asyncio event loop. + + + diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 722ae4593..618b9989e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -216,6 +216,26 @@ Default: ``None`` The name of the region associated with the AWS client. +.. setting:: ASYNCIO_EVENT_LOOP + +ASYNCIO_EVENT_LOOP +------------------ + +Default: ``None`` + +Import path of a given asyncio event loop class. + +If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the +asyncio event loop to be used with it. Set the setting to the import path of the +desired asyncio event loop class. If the setting is set to ``None`` the default asyncio +event loop will be used. + +If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor` +function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop +class to be used. + +Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`. + .. setting:: BOT_NAME BOT_NAME diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d028bea4d..4c6b0e496 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -340,5 +340,5 @@ class CrawlerProcess(CrawlerRunner): def _handle_twisted_reactor(self): if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"]) + install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) super()._handle_twisted_reactor() diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 896afa995..a0251394b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -19,6 +19,8 @@ from os.path import join, abspath, dirname AJAXCRAWL_ENABLED = False +ASYNCIO_EVENT_LOOP = None + AUTOTHROTTLE_ENABLED = False AUTOTHROTTLE_DEBUG = False AUTOTHROTTLE_MAX_DELAY = 60.0 diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 1d6a2c39d..e41315738 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -150,6 +150,13 @@ def log_scrapy_info(settings): logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) + from twisted.internet import asyncioreactor + if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor): + logger.debug( + "Using asyncio event loop: %s.%s", + reactor._asyncioEventloop.__module__, + reactor._asyncioEventloop.__class__.__name__, + ) class StreamLogger: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 3c705f69b..879d27907 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -50,13 +50,19 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) -def install_reactor(reactor_path): +def install_reactor(reactor_path, event_loop_path=None): """Installs the :mod:`~twisted.internet.reactor` with the specified - import path.""" + import path. Also installs the asyncio event loop with the specified import + path if the asyncio reactor is enabled""" reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - asyncioreactor.install(asyncio.get_event_loop()) + if event_loop_path is not None: + event_loop_class = load_object(event_loop_path) + event_loop = event_loop_class() + else: + event_loop = asyncio.new_event_loop() + asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") installer_path = module + ["install"] diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py new file mode 100644 index 000000000..1e4ada722 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -0,0 +1,17 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop" +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index fe1cbc997..44ddcded8 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -13,6 +13,7 @@ pytest-twisted >= 1.11 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures +uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython diff --git a/tests/test_commands.py b/tests/test_commands.py index f76f851e7..ee8a92604 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -16,6 +16,7 @@ from tempfile import mkdtemp from threading import Timer from unittest import skipIf +from pytest import mark from twisted.trial import unittest import scrapy @@ -570,6 +571,28 @@ class BadSpider(scrapy.Spider): log = self.get_log(self.debug_log_spider, args=[]) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + def test_custom_asyncio_loop_enabled_true(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', + 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor', + '-s', + 'ASYNCIO_EVENT_LOOP=uvloop.Loop', + ]) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_custom_asyncio_loop_enabled_false(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' + ]) + import asyncio + loop = asyncio.new_event_loop() + self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log) + def test_output(self): spider_code = """ import scrapy diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 1a4cfe813..7c2e251a9 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -345,6 +345,14 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + def test_custom_loop_asyncio(self): + log = self.run_script("asyncio_custom_loop.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') From a57db9e3024fe1426021b8b692a48f4c8db82a77 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 19 Aug 2020 18:45:24 +0300 Subject: [PATCH 253/568] Bitbucket no longer supports Mercurial repositories (#4738) --- .travis.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index db720b918..33a920bb6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ install: - | if [[ ! -z "$PYPY_VERSION" ]]; then export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" diff --git a/setup.py b/setup.py index d0880051f..52a27c368 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ if has_environment_marker_platform_impl_support(): ] extras_require[':platform_python_implementation == "PyPy"'] = [ # Earlier lxml versions are affected by - # https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no, + # https://foss.heptapod.net/pypy/pypy/-/issues/2498, # which was fixed in Cython 0.26, released on 2017-06-19, and used to # generate the C headers of lxml release tarballs published since then, the # first of which was: From d68aab992e435aa1c88061e83d03e57e7d31533d Mon Sep 17 00:00:00 2001 From: Grisha Temchenko Date: Thu, 20 Aug 2020 09:22:07 -0400 Subject: [PATCH 254/568] Smarter generator check for combined return/yield statements (#4721) --- scrapy/utils/misc.py | 19 ++++++++++++++++++- ...t_return_with_argument_inside_generator.py | 19 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d6966be8e..bd400bd30 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -5,6 +5,7 @@ import os import re import hashlib import warnings +from collections import deque from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules @@ -184,6 +185,22 @@ def set_environ(**kwargs): os.environ[k] = v +def walk_callable(node): + """Similar to ``ast.walk``, but walks only function body and skips nested + functions defined within the node. + """ + todo = deque([node]) + walked_func_def = False + while todo: + node = todo.popleft() + if isinstance(node, ast.FunctionDef): + if walked_func_def: + continue + walked_func_def = True + todo.extend(ast.iter_child_nodes(node)) + yield node + + _generator_callbacks_cache = LocalWeakReferencedCache(limit=128) @@ -201,7 +218,7 @@ def is_generator_with_return_value(callable): if inspect.isgeneratorfunction(callable): tree = ast.parse(dedent(inspect.getsource(callable))) - for node in ast.walk(tree): + for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): _generator_callbacks_cache[callable] = True return _generator_callbacks_cache[callable] diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index bdbec1beb..2be38620c 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -29,9 +29,28 @@ class UtilsMiscPy3TestCase(unittest.TestCase): yield 1 yield from g() + def m(): + yield 1 + + def helper(): + return 0 + + yield helper() + + def n(): + yield 1 + + def helper(): + return 0 + + yield helper() + return 2 + assert is_generator_with_return_value(f) assert is_generator_with_return_value(g) assert not is_generator_with_return_value(h) assert not is_generator_with_return_value(i) assert not is_generator_with_return_value(j) assert not is_generator_with_return_value(k) # not recursive + assert not is_generator_with_return_value(m) + assert is_generator_with_return_value(n) From 2fbfe2c21411480c35a99ff5497ee59f2e5e0dbb Mon Sep 17 00:00:00 2001 From: yogendra0sharma Date: Fri, 21 Aug 2020 12:18:15 +0530 Subject: [PATCH 255/568] Removed appveyor.xml no longer needed --- appveyor.yml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 7fd636864..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,25 +0,0 @@ -platform: x86 -version: '{branch}-{build}' -environment: - matrix: - - PYTHON: "C:\\Python36" - TOX_ENV: py36 - -branches: - only: - - master - - /d+\.\d+\.\d+[\w\-]*$/ - -install: - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%" - - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE" - - "pip install -U tox" - -build: false -skip_tags: true -test_script: - - "tox -e %TOX_ENV%" - -cache: - - '%LOCALAPPDATA%\pip\cache' From e90be0d8a5e3c20ab6aced22ccac59a876db8c99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 21 Aug 2020 14:09:52 +0200 Subject: [PATCH 256/568] Mark the new test as xfail for xmliter_lxml --- tests/test_utils_iterators.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index bbdc88dd1..ae64e36cf 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,5 +1,6 @@ import os +from pytest import mark from twisted.trial import unittest from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml @@ -149,6 +150,26 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('id/text()').getall(), []) self.assertEqual(node.xpath('price/text()').getall(), []) + def test_xmliter_namespaced_nodename(self): + body = b""" + + + + My Dummy Company + http://www.mydummycompany.com + This is a dummy company. We do nothing. + + Item 1 + This is item 1 + http://www.mydummycompany.com/items/1 + http://www.mydummycompany.com/images/item1.jpg + ITEM_1 + 400 + + + + """ + response = XmlResponse(url='http://mydummycompany.com', body=body) my_iter = self.xmliter(response, 'g:image_link') node = next(my_iter) node.register_namespace('g', 'http://base.google.com/ns/1.0') @@ -187,6 +208,10 @@ class XmliterTestCase(unittest.TestCase): class LxmlXmliterTestCase(XmliterTestCase): xmliter = staticmethod(xmliter_lxml) + @mark.xfail(reason='known bug of the current implementation') + def test_xmliter_namespaced_nodename(self): + super().test_xmliter_namespaced_nodename() + def test_xmliter_iterate_namespace(self): body = b""" From afd3a4d116809ecf4c334657c5150c157aa9465b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 21 Aug 2020 17:04:02 +0200 Subject: [PATCH 257/568] Fix style issue --- scrapy/utils/iterators.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c356ad7f8..6f6b5e337 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -41,7 +41,15 @@ def xmliter(obj, nodename): r = re.compile(r'<%(np)s[\s>].*?' % {'np': nodename_patt}, re.DOTALL) for match in r.finditer(text): - nodetext = document_header + match.group().replace(nodename, '%s %s' % (nodename, ' '.join(namespaces.values())), 1) + header_end + nodetext = ( + document_header + + match.group().replace( + nodename, + '%s %s' % (nodename, ' '.join(namespaces.values())), + 1 + ) + + header_end + ) yield Selector(text=nodetext, type='xml') From 7c076122ebb16a6db9b85b68c85c354c2a49fd2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 21 Aug 2020 17:06:54 +0200 Subject: [PATCH 258/568] Skip checks introduced in Pylint 2.6.0 --- pylintrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pylintrc b/pylintrc index 129c7bf7d..5b6b9fab0 100644 --- a/pylintrc +++ b/pylintrc @@ -68,6 +68,7 @@ disable=abstract-method, pointless-statement, pointless-string-statement, protected-access, + raise-missing-from, redefined-argument-from-local, redefined-builtin, redefined-outer-name, @@ -75,6 +76,7 @@ disable=abstract-method, signature-differs, singleton-comparison, super-init-not-called, + super-with-arguments, superfluous-parens, too-few-public-methods, too-many-ancestors, From f1250177dc1c486517b9ca8ed136162533014da7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Sat, 22 Aug 2020 04:33:35 -0300 Subject: [PATCH 259/568] Remove Python 3.5 from CI (#4743) --- .travis.yml | 12 +++--------- azure-pipelines.yml | 4 +--- tests/test_utils_python.py | 18 +++++++++--------- tox.ini | 4 ++-- 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 33a920bb6..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,16 +19,10 @@ matrix: python: 3.8 - env: TOXENV=pinned - python: 3.5.2 + python: 3.6.1 - env: TOXENV=asyncio-pinned - python: 3.5.2 # We use additional code to support 3.5.3 and earlier - - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0 - - - env: TOXENV=py - python: 3.5 - - env: TOXENV=asyncio - python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 - - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 + python: 3.6.1 + - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 - env: TOXENV=py python: 3.6 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 710e42090..c03e258c7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -4,11 +4,9 @@ pool: vmImage: 'windows-latest' strategy: matrix: - Python35: - python.version: '3.5' - TOXENV: windows-pinned Python36: python.version: '3.6' + TOXENV: windows-pinned Python37: python.version: '3.7' Python38: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index c298d0bd2..3115cc92f 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -3,8 +3,8 @@ import gc import operator import platform import unittest +from datetime import datetime from itertools import count -from sys import version_info from warnings import catch_warnings from scrapy.utils.python import ( @@ -216,15 +216,15 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(str.split), []) self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) - else: - self.assertEqual( - get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) - self.assertEqual( - get_func_args(operator.itemgetter(2), stripself=True), ['obj']) - if version_info < (3, 6): - self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) - else: + elif platform.python_implementation() == 'PyPy': + self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) + self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj']) + + build_date = datetime.strptime(platform.python_build()[1], '%b %d %Y') + if build_date >= datetime(2020, 4, 7): # PyPy 3.6-v7.3.1 self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable']) + else: + self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) diff --git a/tox.ini b/tox.ini index 4f5531aea..dec0d75e8 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ deps = # Extras boto3>=1.13.0 botocore>=1.4.87 - Pillow>=3.4.2 + Pillow>=4.0.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -78,7 +78,7 @@ deps = # Extras botocore==1.4.87 google-cloud-storage==1.29.0 - Pillow==3.4.2 + Pillow==4.0.0 [testenv:pinned] deps = From 1432161477673152f4d2f95cd32829a81ffe3f70 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 24 Aug 2020 15:40:01 +0530 Subject: [PATCH 260/568] fix: bump min typing-extensions version to 3.7.4 - typing-extensions>=3.7.4 only supports TypedDict --- setup.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 607f1dadf..39482d383 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.2.0', - 'typing-extensions>=3.7', + 'typing-extensions>=3.7.4', ] extras_require = {} diff --git a/tox.ini b/tox.ini index 3f6fd1224..0a88ed8af 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,7 @@ deps = queuelib==1.4.2 service_identity==16.0.0 Twisted[http2]==17.9.0 - typing-extensions==3.7 + typing-extensions==3.7.4 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt From 2d8ec9d44fa2201986da98c3f5f7c3c7f1ecfb56 Mon Sep 17 00:00:00 2001 From: WinterComes Date: Wed, 17 Jul 2019 22:50:34 +0300 Subject: [PATCH 261/568] Change DOWNLOAD_MAXSIZE logger level from Error to Warning --- scrapy/core/downloader/handlers/http11.py | 21 +++++++++++---------- tests/test_downloader_handlers.py | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index fb04d1fb7..a78b19318 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -394,13 +394,14 @@ class ScrapyAgent: fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss) if maxsize and expected_size > maxsize: - error_msg = ("Cancelling download of %(url)s: expected response " - "size (%(size)s) larger than download max size (%(maxsize)s).") - error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} + warning_msg = ("Expected response size (%(size)s) larger than " + "download max size (%(maxsize)s) in request %(request)s.") + warning_args = {'request': request, 'size': expected_size, 'maxsize': maxsize} + + logger.warning(warning_msg, warning_args) - logger.error(error_msg, error_args) txresponse._transport._producer.loseConnection() - raise defer.CancelledError(error_msg % error_args) + raise defer.CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " @@ -523,11 +524,11 @@ class _ResponseReader(protocol.Protocol): self._finish_response(flags=["download_stopped"], failure=failure) if self._maxsize and self._bytes_received > self._maxsize: - logger.error("Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s) in request %(request)s.", - {'bytes': self._bytes_received, - 'maxsize': self._maxsize, - 'request': self._request}) + logger.warning("Received (%(bytes)s) bytes larger than download " + "max size (%(maxsize)s) in request %(request)s.", + {'bytes': self._bytes_received, + 'maxsize': self._maxsize, + 'request': self._request}) # Clear buffer earlier to avoid keeping data in memory for a long time. self._bodybuf.truncate(0) self._finished.cancel() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 13063d106..7059f0892 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -410,7 +410,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('largechunkedfile')) def check(logger): - logger.error.assert_called_once_with(mock.ANY, mock.ANY) + logger.warning.assert_called_once_with(mock.ANY, mock.ANY) d = self.download_request(request, Spider('foo', download_maxsize=1500)) yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) From 0b3881d65e12e7e0ac5f70c0f33e45dbab546c5d Mon Sep 17 00:00:00 2001 From: drs-11 Date: Mon, 24 Aug 2020 20:26:06 +0530 Subject: [PATCH 262/568] Reverted maxsize warning log message --- scrapy/core/downloader/handlers/http11.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a78b19318..25e800984 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -394,9 +394,9 @@ class ScrapyAgent: fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss) if maxsize and expected_size > maxsize: - warning_msg = ("Expected response size (%(size)s) larger than " - "download max size (%(maxsize)s) in request %(request)s.") - warning_args = {'request': request, 'size': expected_size, 'maxsize': maxsize} + warning_msg = ("Cancelling download of %(url)s: expected response " + "size (%(size)s) larger than download max size (%(maxsize)s).") + warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} logger.warning(warning_msg, warning_args) From 3e726b9df721f2288f4ae9a685de9bdff0762c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Tue, 25 Aug 2020 11:22:05 +0100 Subject: [PATCH 263/568] Support for delegated methods as callbacks It can be useful to structure the spiders code around some helper classes. --- scrapy/utils/reqser.py | 29 ++++++++++++++--------------- tests/test_utils_reqser.py | 12 ++++++++++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 5ea2aafb8..35a4fc72c 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -73,23 +73,22 @@ def request_from_dict(d, spider=None): def _find_method(obj, func): if obj: try: - func_self = func.__self__ - except AttributeError: # func has no __self__ + func.__func__ + except AttributeError: # func is not a instance method. Not supported. pass else: - if func_self is obj: - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name - raise ValueError("Function %s is not a method of: %s" % (func, obj)) + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original + # function object because instance method objects + # are generated each time attribute is retrieved from + # instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name + raise ValueError("Function %s is not an instance method in: %s" % (func, obj)) def _get_method(obj, name): diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index de94ec960..c8d1db138 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -102,6 +102,12 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_delegated_callback_serialization(self): + r = Request("http://www.example.com", + callback=self.spider.delegated_callback, + errback=self.spider.handle_error) + self._assert_serializes_ok(r, spider=self.spider) + def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) @@ -131,6 +137,9 @@ class TestSpiderMixin: def __mixin_callback(self, response): pass +class TestSpiderDelegation: + def delegated_callback(self, response): + pass def parse_item(response): pass @@ -155,6 +164,9 @@ class TestSpider(Spider, TestSpiderMixin): __parse_item_reference = private_parse_item __handle_error_reference = private_handle_error + def __init__(self): + self.delegated_callback = TestSpiderDelegation().delegated_callback + def parse_item(self, response): pass From a2d6fa5adc17035cebb8dad4a3bd2020236b4f71 Mon Sep 17 00:00:00 2001 From: maranqz Date: Tue, 25 Aug 2020 13:34:43 +0300 Subject: [PATCH 264/568] Add errors parameter for CsvItemExporter with tests --- scrapy/exporters.py | 5 +++-- tests/test_exporters.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 95518b3ac..8cd2077b6 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -195,7 +195,7 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): - def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs): + def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs): super().__init__(dont_fail=True, **kwargs) if not self.encoding: self.encoding = 'utf-8' @@ -205,7 +205,8 @@ class CsvItemExporter(BaseItemExporter): line_buffering=False, write_through=True, encoding=self.encoding, - newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + newline='', # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + errors=errors, ) self.csv_writer = csv.writer(self.stream, **self._kwargs) self._headers_not_written = True diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 6c25a0064..ebc477e74 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -355,6 +355,23 @@ class CsvItemExporterTest(BaseItemExporterTest): expected='22,False,3.14,2015-01-01 01:01:01\r\n' ) + def test_errors_default(self): + with self.assertRaises(UnicodeEncodeError): + self.assertExportResult( + item=dict(text=u'W\u0275\u200Brd'), + expected=None, + encoding='windows-1251', + ) + + def test_errors_xmlcharrefreplace(self): + self.assertExportResult( + item=dict(text=u'W\u0275\u200Brd'), + include_headers_line=False, + expected='Wɵ​rd\r\n', + encoding='windows-1251', + errors='xmlcharrefreplace', + ) + class CsvItemExporterDataclassTest(CsvItemExporterTest): item_class = TestDataClass From 39affea93c5b60cde89e000486c39c3c0ce87dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 25 Aug 2020 13:57:48 +0200 Subject: [PATCH 265/568] Fix style issues --- tests/test_utils_reqser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index c8d1db138..ee68cf6b1 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -137,10 +137,12 @@ class TestSpiderMixin: def __mixin_callback(self, response): pass + class TestSpiderDelegation: def delegated_callback(self, response): pass + def parse_item(response): pass From 0524df866936506ae9438a5565fc4733fa5ba5b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Tue, 25 Aug 2020 14:36:38 +0100 Subject: [PATCH 266/568] Code simplification. Thanks @victor-torres for the suggestion --- scrapy/utils/reqser.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 35a4fc72c..503d7b133 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -71,23 +71,19 @@ def request_from_dict(d, spider=None): def _find_method(obj, func): - if obj: - try: - func.__func__ - except AttributeError: # func is not a instance method. Not supported. - pass - else: - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name + # Only instance methods contain ``__func__`` + if obj and hasattr(func, '__func__'): + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original + # function object because instance method objects + # are generated each time attribute is retrieved from + # instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name raise ValueError("Function %s is not an instance method in: %s" % (func, obj)) From 2f28cee3ce482a9380c816b689fc6a5dabc60295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 25 Aug 2020 17:49:17 +0200 Subject: [PATCH 267/568] Add a test to cover searching for a missing node name --- tests/test_utils_iterators.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ae64e36cf..2adccebb8 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -175,6 +175,30 @@ class XmliterTestCase(unittest.TestCase): node.register_namespace('g', 'http://base.google.com/ns/1.0') self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + def test_xmliter_namespaced_nodename_missing(self): + body = b""" + + + + My Dummy Company + http://www.mydummycompany.com + This is a dummy company. We do nothing. + + Item 1 + This is item 1 + http://www.mydummycompany.com/items/1 + http://www.mydummycompany.com/images/item1.jpg + ITEM_1 + 400 + + + + """ + response = XmlResponse(url='http://mydummycompany.com', body=body) + my_iter = self.xmliter(response, 'g:link_image') + with self.assertRaises(StopIteration): + next(my_iter) + def test_xmliter_exception(self): body = ( '' From 58ca8bbf6d1589bd0c8cc1ebda52299346f55e8a Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Sat, 22 Aug 2020 22:32:03 +0200 Subject: [PATCH 268/568] Use f-strings (#4307) --- docs/conf.py | 2 +- docs/intro/tutorial.rst | 6 +- docs/topics/developer-tools.rst | 138 +++++++++--------- docs/topics/exporters.rst | 6 +- docs/topics/item-pipeline.rst | 6 +- docs/topics/leaks.rst | 2 +- docs/topics/selectors.rst | 7 +- docs/topics/settings.rst | 2 +- docs/topics/spiders.rst | 4 +- extras/qps-bench-server.py | 2 +- extras/qpsclient.py | 4 +- scrapy/cmdline.py | 16 +- scrapy/commands/__init__.py | 2 +- scrapy/commands/bench.py | 2 +- scrapy/commands/check.py | 10 +- scrapy/commands/edit.py | 4 +- scrapy/commands/genspider.py | 27 ++-- scrapy/commands/parse.py | 4 +- scrapy/commands/runspider.py | 8 +- scrapy/commands/startproject.py | 12 +- scrapy/commands/version.py | 4 +- scrapy/contracts/__init__.py | 8 +- scrapy/contracts/default.py | 12 +- scrapy/core/downloader/__init__.py | 16 +- scrapy/core/downloader/handlers/__init__.py | 3 +- scrapy/core/downloader/handlers/http11.py | 17 ++- scrapy/core/downloader/handlers/s3.py | 6 +- scrapy/core/downloader/middleware.py | 15 +- scrapy/core/downloader/webclient.py | 6 +- scrapy/core/engine.py | 12 +- scrapy/core/scraper.py | 4 +- scrapy/core/spidermw.py | 20 +-- scrapy/downloadermiddlewares/cookies.py | 12 +- scrapy/downloadermiddlewares/httpproxy.py | 2 +- scrapy/downloadermiddlewares/retry.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 6 +- scrapy/downloadermiddlewares/stats.py | 6 +- scrapy/exporters.py | 2 +- scrapy/extensions/corestats.py | 2 +- scrapy/extensions/debug.py | 2 +- scrapy/extensions/httpcache.py | 10 +- scrapy/extensions/memdebug.py | 2 +- scrapy/extensions/memusage.py | 14 +- scrapy/extensions/statsmailer.py | 10 +- scrapy/http/common.py | 2 +- scrapy/http/headers.py | 2 +- scrapy/http/request/__init__.py | 12 +- scrapy/http/request/form.py | 19 ++- scrapy/http/response/__init__.py | 6 +- scrapy/http/response/text.py | 15 +- scrapy/item.py | 6 +- scrapy/link.py | 6 +- scrapy/logformatter.py | 4 +- scrapy/pipelines/files.py | 23 ++- scrapy/pipelines/images.py | 9 +- scrapy/pipelines/media.py | 7 +- scrapy/pqueues.py | 9 +- scrapy/responsetypes.py | 2 +- scrapy/selector/unified.py | 4 +- scrapy/settings/__init__.py | 2 +- scrapy/settings/default_settings.py | 2 +- scrapy/shell.py | 4 +- scrapy/spiderloader.py | 13 +- scrapy/spidermiddlewares/depth.py | 2 +- scrapy/spidermiddlewares/httperror.py | 2 +- scrapy/spidermiddlewares/offsite.py | 6 +- scrapy/spidermiddlewares/referer.py | 2 +- scrapy/spiders/__init__.py | 11 +- scrapy/spiders/feed.py | 4 +- scrapy/utils/benchserver.py | 6 +- scrapy/utils/conf.py | 22 +-- scrapy/utils/curl.py | 4 +- scrapy/utils/decorators.py | 4 +- scrapy/utils/deprecate.py | 11 +- scrapy/utils/engine.py | 4 +- scrapy/utils/ftp.py | 2 +- scrapy/utils/iterators.py | 13 +- scrapy/utils/log.py | 4 +- scrapy/utils/misc.py | 15 +- scrapy/utils/project.py | 4 +- scrapy/utils/python.py | 12 +- scrapy/utils/reactor.py | 8 +- scrapy/utils/reqser.py | 4 +- scrapy/utils/response.py | 10 +- scrapy/utils/serialize.py | 6 +- scrapy/utils/ssl.py | 4 +- scrapy/utils/test.py | 2 +- scrapy/utils/testproc.py | 6 +- scrapy/utils/testsite.py | 4 +- scrapy/utils/trackref.py | 4 +- scrapy/utils/url.py | 4 +- sep/sep-002.rst | 2 +- sep/sep-004.rst | 4 +- sep/sep-014.rst | 29 ++-- sep/sep-018.rst | 22 +-- tests/CrawlerRunner/ip_address.py | 2 +- tests/mockserver.py | 10 +- tests/py36/_test_crawl.py | 2 +- tests/spiders.py | 8 +- tests/test_cmdline/extensions.py | 2 +- tests/test_command_check.py | 10 +- tests/test_command_parse.py | 16 +- tests/test_command_shell.py | 16 +- tests/test_command_version.py | 2 +- tests/test_commands.py | 32 ++-- tests/test_contracts.py | 2 +- tests/test_crawl.py | 6 +- tests/test_downloader_handlers.py | 22 +-- tests/test_downloadermiddleware.py | 2 +- ...test_downloadermiddleware_decompression.py | 2 +- tests/test_downloadermiddleware_httpcache.py | 10 +- tests/test_downloadermiddleware_redirect.py | 10 +- tests/test_downloadermiddleware_retry.py | 2 +- tests/test_engine.py | 23 +-- tests/test_feedexport.py | 13 +- tests/test_loader_deprecated.py | 2 +- tests/test_logformatter.py | 2 +- tests/test_middleware.py | 2 +- tests/test_pipeline_crawl.py | 4 +- tests/test_pipeline_files.py | 4 +- tests/test_pipeline_images.py | 4 +- tests/test_proxy_connect.py | 6 +- tests/test_request_attribute_binding.py | 4 +- tests/test_responsetypes.py | 12 +- tests/test_selector.py | 2 +- tests/test_signals.py | 2 +- tests/test_spidermiddleware_output_chain.py | 24 +-- tests/test_utils_curl.py | 2 +- tests/test_utils_datatypes.py | 2 +- tests/test_utils_defer.py | 8 +- tests/test_utils_iterators.py | 2 +- tests/test_utils_url.py | 6 +- tests/test_webclient.py | 8 +- 133 files changed, 561 insertions(+), 568 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 427c79481..27d2b5dff 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,7 +49,7 @@ master_doc = 'index' # General information about the project. project = 'Scrapy' -copyright = '2008–{}, Scrapy developers'.format(datetime.now().year) +copyright = f'2008–{datetime.now().year}, Scrapy developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index f96c78887..914b91022 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -101,10 +101,10 @@ This is the code for our first Spider. Save it in a file named def parse(self, response): page = response.url.split("/")[-2] - filename = 'quotes-%s.html' % page + filename = f'quotes-{page}.html' with open(filename, 'wb') as f: f.write(response.body) - self.log('Saved file %s' % filename) + self.log(f'Saved file {filename}') As you can see, our Spider subclasses :class:`scrapy.Spider ` @@ -190,7 +190,7 @@ for your spider:: def parse(self, response): page = response.url.split("/")[-2] - filename = 'quotes-%s.html' % page + filename = f'quotes-{page}.html' with open(filename, 'wb') as f: f.write(response.body) diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 101aa159c..c83b1a9d9 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -5,9 +5,9 @@ Using your browser's Developer Tools for scraping ================================================= Here is a general guide on how to use your browser's Developer Tools -to ease the scraping process. Today almost all browsers come with +to ease the scraping process. Today almost all browsers come with built in `Developer Tools`_ and although we will use Firefox in this -guide, the concepts are applicable to any other browser. +guide, the concepts are applicable to any other browser. In this guide we'll introduce the basic tools to use from a browser's Developer Tools by scraping `quotes.toscrape.com`_. @@ -41,16 +41,16 @@ Therefore, you should keep in mind the following things: Inspecting a website ==================== -By far the most handy feature of the Developer Tools is the `Inspector` -feature, which allows you to inspect the underlying HTML code of -any webpage. To demonstrate the Inspector, let's look at the +By far the most handy feature of the Developer Tools is the `Inspector` +feature, which allows you to inspect the underlying HTML code of +any webpage. To demonstrate the Inspector, let's look at the `quotes.toscrape.com`_-site. On the site we have a total of ten quotes from various authors with specific -tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes -on this page, without any meta-information about authors, tags, etc. +tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes +on this page, without any meta-information about authors, tags, etc. -Instead of viewing the whole source code for the page, we can simply right click +Instead of viewing the whole source code for the page, we can simply right click on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`. In it you should see something like this: @@ -97,16 +97,16 @@ Then, back to your web browser, right-click on the ``span`` tag, select >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] -Adding ``text()`` at the end we are able to extract the first quote with this +Adding ``text()`` at the end we are able to extract the first quote with this basic selector. But this XPath is not really that clever. All it does is -go down a desired path in the source code starting from ``html``. So let's -see if we can refine our XPath a bit: +go down a desired path in the source code starting from ``html``. So let's +see if we can refine our XPath a bit: -If we check the `Inspector` again we'll see that directly beneath our -expanded ``div`` tag we have nine identical ``div`` tags, each with the -same attributes as our first. If we expand any of them, we'll see the same +If we check the `Inspector` again we'll see that directly beneath our +expanded ``div`` tag we have nine identical ``div`` tags, each with the +same attributes as our first. If we expand any of them, we'll see the same structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can -expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and +expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and see each quote: .. code-block:: html @@ -121,7 +121,7 @@ see each quote: With this knowledge we can refine our XPath: Instead of a path to follow, -we'll simply select all ``span`` tags with the ``class="text"`` by using +we'll simply select all ``span`` tags with the ``class="text"`` by using the `has-class-extension`_: >>> response.xpath('//span[has-class("text")]/text()').getall() @@ -130,45 +130,45 @@ the `has-class-extension`_: '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', ...] -And with one simple, cleverer XPath we are able to extract all quotes from -the page. We could have constructed a loop over our first XPath to increase -the number of the last ``div``, but this would have been unnecessarily +And with one simple, cleverer XPath we are able to extract all quotes from +the page. We could have constructed a loop over our first XPath to increase +the number of the last ``div``, but this would have been unnecessarily complex and by simply constructing an XPath with ``has-class("text")`` -we were able to extract all quotes in one line. +we were able to extract all quotes in one line. -The `Inspector` has a lot of other helpful features, such as searching in the +The `Inspector` has a lot of other helpful features, such as searching in the source code or directly scrolling to an element you selected. Let's demonstrate -a use case: +a use case: -Say you want to find the ``Next`` button on the page. Type ``Next`` into the -search bar on the top right of the `Inspector`. You should get two results. -The first is a ``li`` tag with the ``class="next"``, the second the text +Say you want to find the ``Next`` button on the page. Type ``Next`` into the +search bar on the top right of the `Inspector`. You should get two results. +The first is a ``li`` tag with the ``class="next"``, the second the text of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``. If you hover over the tag, you'll see the button highlighted. From here -we could easily create a :ref:`Link Extractor ` to -follow the pagination. On a simple site such as this, there may not be +we could easily create a :ref:`Link Extractor ` to +follow the pagination. On a simple site such as this, there may not be the need to find an element visually but the ``Scroll into View`` function -can be quite useful on complex sites. +can be quite useful on complex sites. Note that the search bar can also be used to search for and test CSS -selectors. For example, you could search for ``span.text`` to find -all quote texts. Instead of a full text search, this searches for -exactly the ``span`` tag with the ``class="text"`` in the page. +selectors. For example, you could search for ``span.text`` to find +all quote texts. Instead of a full text search, this searches for +exactly the ``span`` tag with the ``class="text"`` in the page. .. _topics-network-tool: The Network-tool ================ While scraping you may come across dynamic webpages where some parts -of the page are loaded dynamically through multiple requests. While -this can be quite tricky, the `Network`-tool in the Developer Tools +of the page are loaded dynamically through multiple requests. While +this can be quite tricky, the `Network`-tool in the Developer Tools greatly facilitates this task. To demonstrate the Network-tool, let's -take a look at the page `quotes.toscrape.com/scroll`_. +take a look at the page `quotes.toscrape.com/scroll`_. -The page is quite similar to the basic `quotes.toscrape.com`_-page, -but instead of the above-mentioned ``Next`` button, the page -automatically loads new quotes when you scroll to the bottom. We -could go ahead and try out different XPaths directly, but instead +The page is quite similar to the basic `quotes.toscrape.com`_-page, +but instead of the above-mentioned ``Next`` button, the page +automatically loads new quotes when you scroll to the bottom. We +could go ahead and try out different XPaths directly, but instead we'll check another quite useful command from the Scrapy shell: .. skip: next @@ -179,9 +179,9 @@ we'll check another quite useful command from the Scrapy shell: (...) >>> view(response) -A browser window should open with the webpage but with one -crucial difference: Instead of the quotes we just see a greenish -bar with the word ``Loading...``. +A browser window should open with the webpage but with one +crucial difference: Instead of the quotes we just see a greenish +bar with the word ``Loading...``. .. image:: _images/network_01.png :width: 777 @@ -189,21 +189,21 @@ bar with the word ``Loading...``. :alt: Response from quotes.toscrape.com/scroll The ``view(response)`` command let's us view the response our -shell or later our spider receives from the server. Here we see -that some basic template is loaded which includes the title, +shell or later our spider receives from the server. Here we see +that some basic template is loaded which includes the title, the login-button and the footer, but the quotes are missing. This tells us that the quotes are being loaded from a different request -than ``quotes.toscrape/scroll``. +than ``quotes.toscrape/scroll``. -If you click on the ``Network`` tab, you will probably only see -two entries. The first thing we do is enable persistent logs by -clicking on ``Persist Logs``. If this option is disabled, the +If you click on the ``Network`` tab, you will probably only see +two entries. The first thing we do is enable persistent logs by +clicking on ``Persist Logs``. If this option is disabled, the log is automatically cleared each time you navigate to a different -page. Enabling this option is a good default, since it gives us -control on when to clear the logs. +page. Enabling this option is a good default, since it gives us +control on when to clear the logs. If we reload the page now, you'll see the log get populated with six -new requests. +new requests. .. image:: _images/network_02.png :width: 777 @@ -212,31 +212,31 @@ new requests. Here we see every request that has been made when reloading the page and can inspect each request and its response. So let's find out -where our quotes are coming from: +where our quotes are coming from: -First click on the request with the name ``scroll``. On the right +First click on the request with the name ``scroll``. On the right you can now inspect the request. In ``Headers`` you'll find details about the request headers, such as the URL, the method, the IP-address, and so on. We'll ignore the other tabs and click directly on ``Response``. -What you should see in the ``Preview`` pane is the rendered HTML-code, -that is exactly what we saw when we called ``view(response)`` in the -shell. Accordingly the ``type`` of the request in the log is ``html``. -The other requests have types like ``css`` or ``js``, but what -interests us is the one request called ``quotes?page=1`` with the -type ``json``. +What you should see in the ``Preview`` pane is the rendered HTML-code, +that is exactly what we saw when we called ``view(response)`` in the +shell. Accordingly the ``type`` of the request in the log is ``html``. +The other requests have types like ``css`` or ``js``, but what +interests us is the one request called ``quotes?page=1`` with the +type ``json``. -If we click on this request, we see that the request URL is +If we click on this request, we see that the request URL is ``http://quotes.toscrape.com/api/quotes?page=1`` and the response is a JSON-object that contains our quotes. We can also right-click -on the request and open ``Open in new tab`` to get a better overview. +on the request and open ``Open in new tab`` to get a better overview. .. image:: _images/network_03.png :width: 777 :height: 375 :alt: JSON-object returned from the quotes.toscrape API -With this response we can now easily parse the JSON-object and +With this response we can now easily parse the JSON-object and also request each page to get every quote on the site:: import scrapy @@ -255,17 +255,17 @@ also request each page to get every quote on the site:: yield {"quote": quote["text"]} if data["has_next"]: self.page += 1 - url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page) + url = f"http://quotes.toscrape.com/api/quotes?page={self.page}" yield scrapy.Request(url=url, callback=self.parse) -This spider starts at the first page of the quotes-API. With each -response, we parse the ``response.text`` and assign it to ``data``. -This lets us operate on the JSON-object like on a Python dictionary. +This spider starts at the first page of the quotes-API. With each +response, we parse the ``response.text`` and assign it to ``data``. +This lets us operate on the JSON-object like on a Python dictionary. We iterate through the ``quotes`` and print out the ``quote["text"]``. -If the handy ``has_next`` element is ``true`` (try loading +If the handy ``has_next`` element is ``true`` (try loading `quotes.toscrape.com/api/quotes?page=10`_ in your browser or a -page-number greater than 10), we increment the ``page`` attribute -and ``yield`` a new request, inserting the incremented page-number +page-number greater than 10), we increment the ``page`` attribute +and ``yield`` a new request, inserting the incremented page-number into our ``url``. .. _requests-from-curl: @@ -298,7 +298,7 @@ Note that to translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. As you can see, with a few inspections in the `Network`-tool we -were able to easily replicate the dynamic requests of the scrolling +were able to easily replicate the dynamic requests of the scrolling functionality of the page. Crawling dynamic pages can be quite daunting and pages can be very complex, but it (mostly) boils down to identifying the correct request and replicating it in your spider. diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 11ef5b2a6..793799a9a 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -57,7 +57,7 @@ value of one of their fields:: adapter = ItemAdapter(item) year = adapter['year'] if year not in self.year_to_exporter: - f = open('{}.xml'.format(year), 'wb') + f = open(f'{year}.xml', 'wb') exporter = XmlItemExporter(f) exporter.start_exporting() self.year_to_exporter[year] = exporter @@ -98,7 +98,7 @@ Example:: import scrapy def serialize_price(value): - return '$ %s' % str(value) + return f'$ {str(value)}' class Product(scrapy.Item): name = scrapy.Field() @@ -122,7 +122,7 @@ Example:: def serialize_field(self, field, name, value): if field == 'price': - return '$ %s' % str(value) + return f'$ {str(value)}' return super(Product, self).serialize_field(field, name, value) .. _topics-exporters-reference: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index cd6a6d47e..6287ee0ad 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -96,7 +96,7 @@ contain a price:: adapter['price'] = adapter['price'] * self.vat_factor return item else: - raise DropItem("Missing price in %s" % item) + raise DropItem(f"Missing price in {item}") Write items to a JSON file @@ -211,7 +211,7 @@ item. # Save screenshot to file, filename will be hash of url. url = adapter["url"] url_hash = hashlib.md5(url.encode("utf8")).hexdigest() - filename = "{}.png".format(url_hash) + filename = f"{url_hash}.png" with open(filename, "wb") as f: f.write(response.body) @@ -240,7 +240,7 @@ returns multiples items with the same id:: def process_item(self, item, spider): adapter = ItemAdapter(item) if adapter['id'] in self.ids_seen: - raise DropItem("Duplicate item found: %r" % item) + raise DropItem(f"Duplicate item found: {item!r}") else: self.ids_seen.add(adapter['id']) return item diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index d2f7edf0a..b895b95cb 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -102,7 +102,7 @@ A real example Let's see a concrete example of a hypothetical case of memory leaks. Suppose we have some spider with a line similar to this one:: - return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id, + return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}", callback=self.parse, cb_kwargs={'referer': response}) That line is passing a response reference inside a request which effectively diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 9e2c6ba42..b576fde91 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -328,8 +328,9 @@ too. Here's an example: 'Name: My image 5
    '] >>> for index, link in enumerate(links): -... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get()) -... print('Link number %d points to url %r and image %r' % args) +... href_xpath = link.xpath('@href').get() +... img_xpath = link.xpath('img/@src').get() +... print(f'Link number {index} points to url {href_xpath!r} and image {img_xpath!r}') Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg' Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg' Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg' @@ -822,7 +823,7 @@ with groups of itemscopes and corresponding itemprops:: ... props = scope.xpath(''' ... set:difference(./descendant::*/@itemprop, ... .//*[@itemscope]/*/@itemprop)''') - ... print(" properties: %s" % (props.getall())) + ... print(f" properties: {props.getall()}") ... print("") current scope: ['http://schema.org/Product'] diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 618b9989e..22d60f87c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -110,7 +110,7 @@ In a spider, the settings are available through ``self.settings``:: start_urls = ['http://example.com'] def parse(self, response): - print("Existing settings: %s" % self.settings.attributes.keys()) + print(f"Existing settings: {self.settings.attributes.keys()}") .. note:: The ``settings`` attribute is set in the base Spider class after the spider diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e50e4aa0a..2056664c7 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -279,7 +279,7 @@ Spiders can access arguments in their `__init__` methods:: def __init__(self, category=None, *args, **kwargs): super(MySpider, self).__init__(*args, **kwargs) - self.start_urls = ['http://www.example.com/categories/%s' % category] + self.start_urls = [f'http://www.example.com/categories/{category}'] # ... The default `__init__` method will take any spider arguments @@ -292,7 +292,7 @@ The above example can also be written as follows:: name = 'myspider' def start_requests(self): - yield scrapy.Request('http://www.example.com/categories/%s' % self.category) + yield scrapy.Request(f'http://www.example.com/categories/{self.category}') Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index da7a0022b..a6472b1ba 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -37,7 +37,7 @@ class Root(Resource): if now - self.lastmark >= 3: self.lastmark = now qps = len(self.tail) / sum(self.tail) - print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps)) + print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}') if 'latency' in request.args: latency = float(request.args['latency'][0]) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index fe1f96cbb..f9fb70342 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -37,11 +37,11 @@ class QPSSpider(Spider): def start_requests(self): url = self.benchurl if self.latency is not None: - url += '?latency={0}'.format(self.latency) + url += f'?latency={self.latency}' slots = int(self.slots) if slots > 1: - urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)] + urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)] else: urls = [url] diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 3e88536e4..91482ce01 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -44,7 +44,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'): if inspect.isclass(obj): cmds[entry_point.name] = obj() else: - raise Exception("Invalid entry point %s" % entry_point.name) + raise Exception(f"Invalid entry point {entry_point.name}") return cmds @@ -67,11 +67,11 @@ def _pop_command_name(argv): def _print_header(settings, inproject): + version = scrapy.__version__ if inproject: - print("Scrapy %s - project: %s\n" % (scrapy.__version__, - settings['BOT_NAME'])) + print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n") else: - print("Scrapy %s - no active project\n" % scrapy.__version__) + print(f"Scrapy {version} - no active project\n") def _print_commands(settings, inproject): @@ -81,7 +81,7 @@ def _print_commands(settings, inproject): print("Available commands:") cmds = _get_commands_dict(settings, inproject) for cmdname, cmdclass in sorted(cmds.items()): - print(" %-13s %s" % (cmdname, cmdclass.short_desc())) + print(f" {cmdname:<13} {cmdclass.short_desc()}") if not inproject: print() print(" [ more ] More commands available when run from project directory") @@ -91,7 +91,7 @@ def _print_commands(settings, inproject): def _print_unknown_command(settings, cmdname, inproject): _print_header(settings, inproject) - print("Unknown command: %s\n" % cmdname) + print(f"Unknown command: {cmdname}\n") print('Use "scrapy" to see available commands') @@ -133,7 +133,7 @@ def execute(argv=None, settings=None): sys.exit(2) cmd = cmds[cmdname] - parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax()) + parser.usage = f"scrapy {cmdname} {cmd.syntax()}" parser.description = cmd.long_desc() settings.setdict(cmd.default_settings, priority='command') cmd.settings = settings @@ -155,7 +155,7 @@ def _run_command(cmd, args, opts): def _run_command_profiled(cmd, args, opts): if opts.profile: - sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile) + sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() p = cProfile.Profile() p.runctx('cmd.run(args, opts)', globals(), loc) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index cfd940fe7..23ccffcd9 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -61,7 +61,7 @@ class ScrapyCommand: group.add_option("--logfile", metavar="FILE", help="log file. if omitted stderr will be used") group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help="log level (default: %s)" % self.settings['LOG_LEVEL']) + help=f"log level (default: {self.settings['LOG_LEVEL']})") group.add_option("--nolog", action="store_true", help="disable logging completely") group.add_option("--profile", metavar="FILE", default=None, diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index c9f3b38e0..999c987ea 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider): def start_requests(self): qargs = {'total': self.total, 'show': self.show} - url = '{}?{}'.format(self.baseurl, urlencode(qargs, doseq=1)) + url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}' return [scrapy.Request(url, dont_filter=True)] def parse(self, response): diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 09a76ca7a..7e848dc97 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult): plural = "s" if run != 1 else "" writeln(self.separator2) - writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start)) + writeln(f"Ran {run} contract{plural} in {stop - start:.3f}") writeln() infos = [] @@ -25,14 +25,14 @@ class TextTestResult(_TextTestResult): write("FAILED") failed, errored = map(len, (self.failures, self.errors)) if failed: - infos.append("failures=%d" % failed) + infos.append(f"failures={failed}") if errored: - infos.append("errors=%d" % errored) + infos.append(f"errors={errored}") else: write("OK") if infos: - writeln(" (%s)" % (", ".join(infos),)) + writeln(f" ({', '.join(infos)})") else: write("\n") @@ -85,7 +85,7 @@ class Command(ScrapyCommand): continue print(spider) for method in sorted(methods): - print(' * %s' % method) + print(f' * {method}') else: start = time.time() self.crawler_process.start() diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 25d843a53..177b20143 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -32,8 +32,8 @@ class Command(ScrapyCommand): try: spidercls = self.crawler_process.spider_loader.load(args[0]) except KeyError: - return self._err("Spider not found: %s" % args[0]) + return self._err(f"Spider not found: {args[0]}") sfile = sys.modules[spidercls.__module__].__file__ sfile = sfile.replace('.pyc', '.py') - self.exitcode = os.system('%s "%s"' % (editor, sfile)) + self.exitcode = os.system(f'{editor} "{sfile}"') diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 74a077d1b..72248bded 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -73,17 +73,18 @@ class Command(ScrapyCommand): if template_file: self._genspider(module, name, domain, opts.template, template_file) if opts.edit: - self.exitcode = os.system('scrapy edit "%s"' % name) + self.exitcode = os.system(f'scrapy edit "{name}"') def _genspider(self, module, name, domain, template_name, template_file): """Generate the spider module, based on the given template""" + capitalized_module = ''.join(s.capitalize() for s in module.split('_')) tvars = { 'project_name': self.settings.get('BOT_NAME'), 'ProjectName': string_camelcase(self.settings.get('BOT_NAME')), 'module': module, 'name': name, 'domain': domain, - 'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_')) + 'classname': f'{capitalized_module}Spider' } if self.settings.get('NEWSPIDER_MODULE'): spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) @@ -91,32 +92,32 @@ class Command(ScrapyCommand): else: spiders_module = None spiders_dir = "." - spider_file = "%s.py" % join(spiders_dir, module) + spider_file = f"{join(spiders_dir, module)}.py" shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - print("Created spider %r using template %r " - % (name, template_name), end=('' if spiders_module else '\n')) + print(f"Created spider {name!r} using template {template_name!r} ", + end=('' if spiders_module else '\n')) if spiders_module: - print("in module:\n %s.%s" % (spiders_module.__name__, module)) + print("in module:\n {spiders_module.__name__}.{module}") def _find_template(self, template): - template_file = join(self.templates_dir, '%s.tmpl' % template) + template_file = join(self.templates_dir, f'{template}.tmpl') if exists(template_file): return template_file - print("Unable to find template: %s\n" % template) + print(f"Unable to find template: {template}\n") print('Use "scrapy genspider --list" to see all available templates.') def _list_templates(self): print("Available templates:") for filename in sorted(os.listdir(self.templates_dir)): if filename.endswith('.tmpl'): - print(" %s" % splitext(filename)[0]) + print(f" {splitext(filename)[0]}") def _spider_exists(self, name): if not self.settings.get('NEWSPIDER_MODULE'): # if run as a standalone command and file with same filename already exists if exists(name + ".py"): - print("%s already exists" % (abspath(name + ".py"))) + print(f"{abspath(name + '.py')} already exists") return True return False @@ -126,8 +127,8 @@ class Command(ScrapyCommand): pass else: # if spider with same name exists - print("Spider %r already exists in module:" % name) - print(" %s" % spidercls.__module__) + print(f"Spider {name!r} already exists in module:") + print(f" {spidercls.__module__}") return True # a file with the same name exists in the target directory @@ -135,7 +136,7 @@ class Command(ScrapyCommand): spiders_dir = dirname(spiders_module.__file__) spiders_dir_abs = abspath(spiders_dir) if exists(join(spiders_dir_abs, name + ".py")): - print("%s already exists" % (join(spiders_dir_abs, (name + ".py")))) + print(f"{join(spiders_dir_abs, (name + '.py'))} already exists") return True return False diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index abc8ba9ff..83ee074da 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -96,13 +96,13 @@ class Command(BaseRunSpiderCommand): if opts.verbose: for level in range(1, self.max_level + 1): - print('\n>>> DEPTH LEVEL: %s <<<' % level) + print(f'\n>>> DEPTH LEVEL: {level} <<<') if not opts.noitems: self.print_items(level, colour) if not opts.nolinks: self.print_requests(level, colour) else: - print('\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level) + print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<') if not opts.noitems: self.print_items(colour=colour) if not opts.nolinks: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index befee021b..aedd8c2ce 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -12,7 +12,7 @@ def _import_file(filepath): dirname, file = os.path.split(abspath) fname, fext = os.path.splitext(file) if fext != '.py': - raise ValueError("Not a Python source file: %s" % abspath) + raise ValueError(f"Not a Python source file: {abspath}") if dirname: sys.path = [dirname] + sys.path try: @@ -42,14 +42,14 @@ class Command(BaseRunSpiderCommand): raise UsageError() filename = args[0] if not os.path.exists(filename): - raise UsageError("File not found: %s\n" % filename) + raise UsageError(f"File not found: {filename}\n") try: module = _import_file(filename) except (ImportError, ValueError) as e: - raise UsageError("Unable to load %r: %s\n" % (filename, e)) + raise UsageError(f"Unable to load {filename!r}: {e}\n") spclasses = list(iter_spider_classes(module)) if not spclasses: - raise UsageError("No spider found in file: %s\n" % filename) + raise UsageError(f"No spider found in file: {filename}\n") spidercls = spclasses.pop() self.crawler_process.crawl(spidercls, **opts.spargs) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e5158d993..1d73fa0cb 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -52,7 +52,7 @@ class Command(ScrapyCommand): print('Error: Project names must begin with a letter and contain' ' only\nletters, numbers and underscores') elif _module_exists(project_name): - print('Error: Module %r already exists' % project_name) + print(f'Error: Module {project_name!r} already exists') else: return True return False @@ -100,7 +100,7 @@ class Command(ScrapyCommand): if exists(join(project_dir, 'scrapy.cfg')): self.exitcode = 1 - print('Error: scrapy.cfg already exists in %s' % abspath(project_dir)) + print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}') return if not self._is_valid_name(project_name): @@ -113,11 +113,11 @@ class Command(ScrapyCommand): path = join(*paths) tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name)) render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) - print("New Scrapy project '%s', using template directory '%s', " - "created in:" % (project_name, self.templates_dir)) - print(" %s\n" % abspath(project_dir)) + print(f"New Scrapy project '{project_name}', using template directory " + f"'{self.templates_dir}', created in:") + print(f" {abspath(project_dir)}\n") print("You can start your first spider with:") - print(" cd %s" % project_dir) + print(f" cd {project_dir}") print(" scrapy genspider example example.com") @property diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index d0ea72a67..dc8087043 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -23,8 +23,8 @@ class Command(ScrapyCommand): if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) - patt = "%-{}s : %s".format(width) + patt = f"%-{width}s : %s" for name, version in versions: print(patt % (name, version)) else: - print("Scrapy %s" % scrapy.__version__) + print(f"Scrapy {scrapy.__version__}") diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 5af3831a2..db0a56e56 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -112,8 +112,8 @@ class Contract: request_cls = None def __init__(self, method, *args): - self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name) - self.testcase_post = _create_testcase(method, '@%s post-hook' % self.name) + self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') + self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') self.args = args def add_pre_hook(self, request, results): @@ -172,8 +172,8 @@ def _create_testcase(method, desc): class ContractTestCase(TestCase): def __str__(_self): - return "[%s] %s (%s)" % (spider, method.__name__, desc) + return f"[{spider}] {method.__name__} ({desc})" - name = '%s_%s' % (spider, method.__name__) + name = f'{spider}_{method.__name__}' setattr(ContractTestCase, name, lambda x: x) return ContractTestCase(name) diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index cfdcc7c25..9704f5253 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -60,8 +60,7 @@ class ReturnsContract(Contract): if len(self.args) not in [1, 2, 3]: raise ValueError( - "Incorrect argument quantity: expected 1, 2 or 3, got %i" - % len(self.args) + f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}" ) self.obj_name = self.args[0] or None self.obj_type_verifier = self.object_type_verifiers[self.obj_name] @@ -88,10 +87,9 @@ class ReturnsContract(Contract): if self.min_bound == self.max_bound: expected = self.min_bound else: - expected = '%s..%s' % (self.min_bound, self.max_bound) + expected = f'{self.min_bound}..{self.max_bound}' - raise ContractFail("Returned %s %s, expected %s" % - (occurrences, self.obj_name, expected)) + raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}") class ScrapesContract(Contract): @@ -106,5 +104,5 @@ class ScrapesContract(Contract): if is_item(x): missing = [arg for arg in self.args if arg not in ItemAdapter(x)] if missing: - missing_str = ", ".join(missing) - raise ContractFail("Missing fields: %s" % missing_str) + missing_fields = ", ".join(missing) + raise ContractFail(f"Missing fields: {missing_fields}") diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index dc5cf1ab8..12a9db6dd 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -41,17 +41,17 @@ class Slot: def __repr__(self): cls_name = self.__class__.__name__ - return "%s(concurrency=%r, delay=%0.2f, randomize_delay=%r)" % ( - cls_name, self.concurrency, self.delay, self.randomize_delay) + return (f"{cls_name}(concurrency={self.concurrency!r}, " + f"delay={self.delay:.2f}, " + f"randomize_delay={self.randomize_delay!r}") def __str__(self): return ( - "" % ( - self.concurrency, self.delay, self.randomize_delay, - len(self.active), len(self.queue), len(self.transferring), - datetime.fromtimestamp(self.lastseen).isoformat() - ) + f"" ) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index e86680978..73aeb2352 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -71,8 +71,7 @@ class DownloadHandlers: scheme = urlparse_cached(request).scheme handler = self._get_handler(scheme) if not handler: - raise NotSupported("Unsupported URL scheme '%s': %s" % - (scheme, self._notconfigured[scheme])) + raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}") return handler.download_request(request, spider) @defer.inlineCallbacks diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 25e800984..1b041c8a8 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -60,11 +60,11 @@ class HTTP11DownloadHandler: settings=settings, crawler=crawler, ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + msg = f""" + '{settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]}' does not accept `method` \ + argument (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD) and/or \ + `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" warnings.warn(msg) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') @@ -169,8 +169,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): else: extra = rcvd_bytes[:32] self._tunnelReadyDeferred.errback( - TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % ( - self._host, self._port, extra))) + TunnelError('Could not open CONNECT tunnel with proxy ' + f'{self._host}:{self._port} [{extra!r}]') + ) def connectFailed(self, reason): """Propagates the errback to the appropriate deferred.""" @@ -371,7 +372,7 @@ class ScrapyAgent: if self._txresponse: self._txresponse._transport.stopProducing() - raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout)) + raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") def _cb_latency(self, result, request, start_time): request.meta['download_latency'] = time() - start_time diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 8f63ad974..0ef977893 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -56,7 +56,7 @@ class S3DownloadHandler: import botocore.credentials kw.pop('anon', None) if kw: - raise TypeError('Unexpected keyword arguments: %s' % kw) + raise TypeError(f'Unexpected keyword arguments: {kw}') if not self.anon: SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] self._signer = SignerCls(botocore.credentials.Credentials( @@ -85,14 +85,14 @@ class S3DownloadHandler: scheme = 'https' if request.meta.get('is_secure') else 'http' bucket = p.hostname path = p.path + '?' + p.query if p.query else p.path - url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path) + url = f'{scheme}://{bucket}.s3.amazonaws.com{path}' if self.anon: request = request.replace(url=url) elif self._signer is not None: import botocore.awsrequest awsrequest = botocore.awsrequest.AWSRequest( method=request.method, - url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path), + url=f'{scheme}://s3.amazonaws.com/{bucket}{path}', headers=request.headers.to_unicode_dict(), data=request.body) self._signer.add_auth(awsrequest) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 4c2eea522..b0e612e43 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -36,8 +36,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - "Middleware %s.process_request must return None, Response or Request, got %s" - % (method.__self__.__class__.__name__, response.__class__.__name__) + f"Middleware {method.__self__.__class__.__name__}" + ".process_request must return None, Response or " + f"Request, got {response.__class__.__name__}" ) if response: return response @@ -54,8 +55,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( - "Middleware %s.process_response must return Response or Request, got %s" - % (method.__self__.__class__.__name__, type(response)) + f"Middleware {method.__self__.__class__.__name__}" + ".process_response must return Response or Request, " + f"got {type(response)}" ) if isinstance(response, Request): return response @@ -68,8 +70,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - "Middleware %s.process_exception must return None, Response or Request, got %s" - % (method.__self__.__class__.__name__, type(response)) + f"Middleware {method.__self__.__class__.__name__}" + ".process_exception must return None, Response or " + f"Request, got {type(response)}" ) if response: return response diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index b2b96f1ea..c13683393 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient): self.transport.stopProducing() self.factory.noPage( - defer.TimeoutError("Getting %s took longer than %s seconds." - % (self.factory.url, self.factory.timeout))) + defer.TimeoutError(f"Getting {self.factory.url} took longer " + f"than {self.factory.timeout} seconds.")) # This class used to inherit from Twisted’s @@ -155,7 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory): self.headers['Content-Length'] = 0 def __repr__(self): - return "<%s: %s>" % (self.__class__.__name__, self.url) + return f"<{self.__class__.__name__}: {self.url}>" def _cancelTimeout(self, result, timeoutCall): if timeoutCall.active(): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 5e0dfe37c..93bcdb49a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -171,8 +171,8 @@ class ExecutionEngine: def _handle_downloader_output(self, response, request, spider): if not isinstance(response, (Request, Response, Failure)): raise TypeError( - "Incorrect type: expected Request, Response or Failure, got %s: %r" - % (type(response), response) + "Incorrect type: expected Request, Response or Failure, got " + f"{type(response)}: {response!r}" ) # downloader middleware can return requests (for example, redirects) if isinstance(response, Request): @@ -214,7 +214,7 @@ class ExecutionEngine: def crawl(self, request, spider): if spider not in self.open_spiders: - raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request)) + raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}") self.schedule(request, spider) self.slot.nextcall.schedule() @@ -239,8 +239,8 @@ class ExecutionEngine: def _on_success(response): if not isinstance(response, (Response, Request)): raise TypeError( - "Incorrect type: expected Response or Request, got %s: %r" - % (type(response), response) + "Incorrect type: expected Response or Request, got " + f"{type(response)}: {response!r}" ) if isinstance(response, Response): if response.request is None: @@ -268,7 +268,7 @@ class ExecutionEngine: @defer.inlineCallbacks def open_spider(self, spider, start_requests=(), close_if_idle=True): if not self.has_capacity(): - raise RuntimeError("No free spider slot when opening %r" % spider.name) + raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={'spider': spider}) nextcall = CallLaterOnce(self._next_request, spider) scheduler = self.scheduler_cls.from_crawler(self.crawler) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 20bdb22a1..0d3e3450f 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -125,7 +125,7 @@ class Scraper: Handle the downloaded response or failure through the spider callback/errback """ if not isinstance(result, (Response, Failure)): - raise TypeError("Incorrect type: expected Response or Failure, got %s: %r" % (type(result), result)) + raise TypeError(f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}") dfd = self._scrape2(result, request, spider) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) dfd.addCallback(self.handle_spider_output, request, result, spider) @@ -173,7 +173,7 @@ class Scraper: spider=spider ) self.crawler.stats.inc_value( - "spider_exceptions/%s" % _failure.value.__class__.__name__, + f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider ) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 5a99b96be..763e0cdf6 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -19,10 +19,7 @@ def _isiterable(possible_iterator): def _fname(f): - return "{}.{}".format( - f.__self__.__class__.__name__, - f.__func__.__name__ - ) + return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" class SpiderMiddlewareManager(MiddlewareManager): @@ -51,8 +48,9 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - msg = "Middleware {} must return None or raise an exception, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = (f"Middleware {_fname(method)} must return None " + f"or raise an exception, got {type(result)}") + raise _InvalidOutput(msg) except _InvalidOutput: raise except Exception: @@ -86,8 +84,9 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = "Middleware {} must return None or an iterable, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = (f"Middleware {_fname(method)} must return None " + f"or an iterable, got {type(result)}") + raise _InvalidOutput(msg) return _failure def process_spider_output(result, start_index=0): @@ -110,8 +109,9 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = _evaluate_iterable(result, method_index + 1, recovered) else: - msg = "Middleware {} must return an iterable, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = (f"Middleware {_fname(method)} must return an " + f"iterable, got {type(result)}") + raise _InvalidOutput(msg) return MutableChain(result, recovered) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 77048f389..e2b7dd901 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -54,8 +54,8 @@ class CookiesMiddleware: cl = [to_unicode(c, errors='replace') for c in request.headers.getlist('Cookie')] if cl: - cookies = "\n".join("Cookie: {}\n".format(c) for c in cl) - msg = "Sending cookies to: {}\n{}".format(request, cookies) + cookies = "\n".join(f"Cookie: {c}\n" for c in cl) + msg = f"Sending cookies to: {request}\n{cookies}" logger.debug(msg, extra={'spider': spider}) def _debug_set_cookie(self, response, spider): @@ -63,8 +63,8 @@ class CookiesMiddleware: cl = [to_unicode(c, errors='replace') for c in response.headers.getlist('Set-Cookie')] if cl: - cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl) - msg = "Received cookies from: {}\n{}".format(response, cookies) + cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl) + msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={'spider': spider}) def _format_cookie(self, cookie, request): @@ -90,9 +90,9 @@ class CookiesMiddleware: request, cookie) decoded[key] = cookie[key].decode("latin1", errors="replace") - cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value")) + cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" for key, value in decoded.items(): # path, domain - cookie_str += "; {}={}".format(key.capitalize(), value) + cookie_str += f"; {key.capitalize()}={value}" return cookie_str def _get_request_cookies(self, jar, request): diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index da89d3e9b..04da11311 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -24,7 +24,7 @@ class HttpProxyMiddleware: def _basic_auth_header(self, username, password): user_pass = to_bytes( - '%s:%s' % (unquote(username), unquote(password)), + f'{unquote(username)}:{unquote(password)}', encoding=self.auth_encoding) return base64.b64encode(user_pass) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 67be8c282..51fe59254 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -88,7 +88,7 @@ class RetryMiddleware: reason = global_object_name(reason.__class__) stats.inc_value('retry/count') - stats.inc_value('retry/reason_count/%s' % reason) + stats.inc_value(f'retry/reason_count/{reason}') return retryreq else: stats.inc_value('retry/max_reached') diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7f18b2bf2..d6da55535 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -61,7 +61,7 @@ class RobotsTxtMiddleware: if netloc not in self._parsers: self._parsers[netloc] = Deferred() - robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc) + robotsurl = f"{url.scheme}://{url.netloc}/robots.txt" robotsreq = Request( robotsurl, priority=self.DOWNLOAD_PRIORITY, @@ -94,7 +94,7 @@ class RobotsTxtMiddleware: def _parse_robots(self, response, netloc, spider): self.crawler.stats.inc_value('robotstxt/response_count') - self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status)) + self.crawler.stats.inc_value(f'robotstxt/response_status_count/{response.status}') rp = self._parserimpl.from_crawler(self.crawler, response.body) rp_dfd = self._parsers[netloc] self._parsers[netloc] = rp @@ -102,7 +102,7 @@ class RobotsTxtMiddleware: def _robots_error(self, failure, netloc): if failure.type is not IgnoreRequest: - key = 'robotstxt/exception_count/{}'.format(failure.type) + key = f'robotstxt/exception_count/{failure.type}' self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] self._parsers[netloc] = None diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 46a2ad397..5479cd0e2 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -17,13 +17,13 @@ class DownloaderStats: def process_request(self, request, spider): self.stats.inc_value('downloader/request_count', spider=spider) - self.stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider) + self.stats.inc_value(f'downloader/request_method_count/{request.method}', spider=spider) reqlen = len(request_httprepr(request)) self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider) def process_response(self, request, response, spider): self.stats.inc_value('downloader/response_count', spider=spider) - self.stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider) + self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider) reslen = len(response_httprepr(response)) self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) return response @@ -31,4 +31,4 @@ class DownloaderStats: def process_exception(self, request, exception, spider): ex_class = global_object_name(exception.__class__) self.stats.inc_value('downloader/exception_count', spider=spider) - self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider) + self.stats.inc_value(f'downloader/exception_type_count/{ex_class}', spider=spider) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 95518b3ac..54cf5c0b1 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -39,7 +39,7 @@ class BaseItemExporter: self.export_empty_fields = options.pop('export_empty_fields', False) self.indent = options.pop('indent', None) if not dont_fail and options: - raise TypeError("Unexpected options: %s" % ', '.join(options.keys())) + raise TypeError(f"Unexpected options: {', '.join(options.keys())}") def export_item(self, item): raise NotImplementedError diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 389cb65bc..675f8276f 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -43,4 +43,4 @@ class CoreStats: def item_dropped(self, item, spider, exception): reason = exception.__class__.__name__ self.stats.inc_value('item_dropped_count', spider=spider) - self.stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider) + self.stats.inc_value(f'item_dropped_reasons_count/{reason}', spider=spider) diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 586399784..fd2a02d8d 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -48,7 +48,7 @@ class StackTraceDump: for id_, frame in sys._current_frames().items(): name = id2name.get(id_, '') dump = ''.join(traceback.format_stack(frame)) - dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump) + dumps += f"# Thread: {name}({id_})\n{dump}\n" return dumps diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 6294a9b52..e0c04b2de 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -223,7 +223,7 @@ class DbmCacheStorage: self.db = None def open_spider(self, spider): - dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) + dbpath = os.path.join(self.cachedir, f'{spider.name}.db') self.db = self.dbmodule.open(dbpath, 'c') logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) @@ -251,13 +251,13 @@ class DbmCacheStorage: 'headers': dict(response.headers), 'body': response.body, } - self.db['%s_data' % key] = pickle.dumps(data, protocol=4) - self.db['%s_time' % key] = str(time()) + self.db[f'{key}_data'] = pickle.dumps(data, protocol=4) + self.db[f'{key}_time'] = str(time()) def _read_data(self, spider, request): key = self._request_key(request) db = self.db - tkey = '%s_time' % key + tkey = f'{key}_time' if tkey not in db: return # not found @@ -265,7 +265,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return # expired - return pickle.loads(db['%s_data' % key]) + return pickle.loads(db[f'{key}_data']) def _request_key(self, request): return request_fingerprint(request) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index dc8cdbb1d..cee44ea62 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -30,4 +30,4 @@ class MemoryDebugger: for cls, wdict in live_refs.items(): if not wdict: continue - self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider) + self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index ab2e43e8c..274cbdbfe 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -82,8 +82,8 @@ class MemoryUsage: {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( - "%s terminated: memory usage exceeded %dM at %s" - % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + f"{self.crawler.settings['BOT_NAME']} terminated: " + f"memory usage exceeded {mem}M at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) @@ -105,8 +105,8 @@ class MemoryUsage: {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( - "%s warning: memory usage reached %dM at %s" - % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + f"{self.crawler.settings['BOT_NAME']} warning: " + f"memory usage reached {mem}M at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/warning_notified', 1) @@ -115,9 +115,9 @@ class MemoryUsage: def _send_report(self, rcpts, subject): """send notification mail with some additional useful info""" stats = self.crawler.stats - s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024) - s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024) - s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024) + s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n" + s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n" + s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n" s += "ENGINE STATUS ------------------------------------------------------- \r\n" s += "\r\n" diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 320f13b29..997e74fc9 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -24,11 +24,11 @@ class StatsMailer: o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - + def spider_closed(self, spider): spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" - body += "\n".join("%-50s : %s" % i for i in self.stats.get_stats().items()) - body += "\n\n%s stats\n\n" % spider.name - body += "\n".join("%-50s : %s" % i for i in spider_stats.items()) - return self.mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body) + body += "\n".join(f"{i:<50} : {self.stats.get_stats()[i]}" for i in self.stats.get_stats()) + body += f"\n\n{spider.name} stats\n\n" + body += "\n".join(f"{i:<50} : {spider_stats[i]}" for i in spider_stats) + return self.mail.send(self.recipients, f"Scrapy stats for: {spider.name}", body) diff --git a/scrapy/http/common.py b/scrapy/http/common.py index ba6ab277c..98699d7fd 100644 --- a/scrapy/http/common.py +++ b/scrapy/http/common.py @@ -1,6 +1,6 @@ def obsolete_setter(setter, attrname): def newsetter(self, value): c = self.__class__.__name__ - msg = "%s.%s is not modifiable, use %s.replace() instead" % (c, attrname, c) + msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead" raise AttributeError(msg) return newsetter diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 6bf9e5346..1a2b99b0a 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -33,7 +33,7 @@ class Headers(CaselessDict): elif isinstance(x, int): return str(x).encode(self.encoding) else: - raise TypeError('Unsupported value type: {}'.format(type(x))) + raise TypeError(f'Unsupported value type: {type(x)}') def __getitem__(self, key): try: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a98ba9960..ef58deacc 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -25,13 +25,13 @@ class Request(object_ref): self._set_url(url) self._set_body(body) if not isinstance(priority, int): - raise TypeError("Request priority not an integer: %r" % priority) + raise TypeError(f"Request priority not an integer: {priority!r}") self.priority = priority if callback is not None and not callable(callback): - raise TypeError('callback must be a callable, got %s' % type(callback).__name__) + raise TypeError(f'callback must be a callable, got {type(callback).__name__}') if errback is not None and not callable(errback): - raise TypeError('errback must be a callable, got %s' % type(errback).__name__) + raise TypeError(f'errback must be a callable, got {type(errback).__name__}') self.callback = callback self.errback = errback @@ -60,13 +60,13 @@ class Request(object_ref): def _set_url(self, url): if not isinstance(url, str): - raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) + raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}') s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) if ('://' not in self._url) and (not self._url.startswith('data:')): - raise ValueError('Missing scheme in request url: %s' % self._url) + raise ValueError(f'Missing scheme in request url: {self._url}') url = property(_get_url, obsolete_setter(_set_url, 'url')) @@ -86,7 +86,7 @@ class Request(object_ref): return self._encoding def __str__(self): - return "<%s %s>" % (self.method, self.url) + return f"<{self.method} {self.url}>" __repr__ = __str__ diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 59af81321..c90d68fa1 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -80,15 +80,15 @@ def _get_form(response, formname, formid, formnumber, formxpath): base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: - raise ValueError("No
    element found in %s" % response) + raise ValueError(f"No element found in {response}") if formname is not None: - f = root.xpath('//form[@name="%s"]' % formname) + f = root.xpath(f'//form[@name="{formname}"]') if f: return f[0] if formid is not None: - f = root.xpath('//form[@id="%s"]' % formid) + f = root.xpath(f'//form[@id="{formid}"]') if f: return f[0] @@ -103,7 +103,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): el = el.getparent() if el is None: break - raise ValueError('No element found with %s' % formxpath) + raise ValueError(f'No element found with {formxpath}') # If we get here, it means that either formname was None # or invalid @@ -111,8 +111,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): try: form = forms[formnumber] except IndexError: - raise IndexError("Form number %d not found in %s" % - (formnumber, response)) + raise IndexError(f"Form number {formnumber} not found in {response}") else: return form @@ -205,12 +204,12 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items()) + xpath = './/*' + ''.join(f'[@{key}="{clickdata[key]}"]' for key in clickdata) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') elif len(el) > 1: - raise ValueError("Multiple elements found (%r) matching the criteria " - "in clickdata: %r" % (el, clickdata)) + raise ValueError(f"Multiple elements found ({el!r}) matching the " + f"criteria in clickdata: {clickdata!r}") else: - raise ValueError('No clickable element matching clickdata: %r' % (clickdata,)) + raise ValueError(f'No clickable element matching clickdata: {clickdata!r}') diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c2c37dd1d..c635fde69 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -55,8 +55,8 @@ class Response(object_ref): if isinstance(url, str): self._url = url else: - raise TypeError('%s url must be str, got %s:' % - (type(self).__name__, type(url).__name__)) + raise TypeError(f'{type(self).__name__} url must be str, ' + f'got {type(url).__name__}') url = property(_get_url, obsolete_setter(_set_url, 'url')) @@ -77,7 +77,7 @@ class Response(object_ref): body = property(_get_body, obsolete_setter(_set_body, 'body')) def __str__(self): - return "<%d %s>" % (self.status, self.url) + return f"<{self.status} {self.url}>" __repr__ = __str__ diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index a7bb34d48..e36e14880 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -47,8 +47,8 @@ class TextResponse(Response): self._body = b'' # used by encoding detection if isinstance(body, str): if self._encoding is None: - raise TypeError('Cannot convert unicode body - %s has no encoding' % - type(self).__name__) + raise TypeError('Cannot convert unicode body - ' + f'{type(self).__name__} has no encoding') self._body = body.encode(self._encoding) else: super()._set_body(body) @@ -92,7 +92,7 @@ class TextResponse(Response): # _body_inferred_encoding is called benc = self.encoding if self._cached_ubody is None: - charset = 'charset=%s' % benc + charset = f'charset={benc}' self._cached_ubody = html_to_unicode(charset, self.body)[1] return self._cached_ubody @@ -255,12 +255,11 @@ def _url_from_selector(sel): # e.g. ::attr(href) result return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): - raise _InvalidSelector("Unsupported selector: %s" % sel) + raise _InvalidSelector(f"Unsupported selector: {sel}") if sel.root.tag not in ('a', 'link'): - raise _InvalidSelector("Only and elements are supported; got <%s>" % - sel.root.tag) + raise _InvalidSelector("Only and elements are supported; " + f"got <{sel.root.tag}>") href = sel.root.get('href') if href is None: - raise _InvalidSelector("<%s> element has no href attribute: %s" % - (sel.root.tag, sel)) + raise _InvalidSelector(f"<{sel.root.tag}> element has no href attribute: {sel}") return strip_html5_whitespace(href) diff --git a/scrapy/item.py b/scrapy/item.py index c262a153c..af3849302 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -96,19 +96,19 @@ class DictItem(MutableMapping, BaseItem): if key in self.fields: self._values[key] = value else: - raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key)) + raise KeyError(f"{self.__class__.__name__} does not support field: {key}") def __delitem__(self, key): del self._values[key] def __getattr__(self, name): if name in self.fields: - raise AttributeError("Use item[%r] to get field value" % name) + raise AttributeError(f"Use item[{name!r}] to get field value") raise AttributeError(name) def __setattr__(self, name, value): if not name.startswith('_'): - raise AttributeError("Use item[%r] = %r to set field value" % (name, value)) + raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value") super().__setattr__(name, value) def __len__(self): diff --git a/scrapy/link.py b/scrapy/link.py index 1ef50b113..684735f6e 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -14,7 +14,7 @@ class Link: def __init__(self, url, text='', fragment='', nofollow=False): if not isinstance(url, str): got = url.__class__.__name__ - raise TypeError("Link urls must be str objects, got %s" % got) + raise TypeError(f"Link urls must be str objects, got {got}") self.url = url self.text = text self.fragment = fragment @@ -33,6 +33,6 @@ class Link: def __repr__(self): return ( - 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' - % (self.url, self.text, self.fragment, self.nofollow) + f'Link(url={self.url!r}, text={self.text!r}, ' + f'fragment={self.fragment!r}, nofollow={self.nofollow!r})' ) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 0f9e6f1cb..87568b2d1 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -54,8 +54,8 @@ class LogFormatter: def crawled(self, request, response, spider): """Logs a message when the crawler finds a webpage.""" - request_flags = ' %s' % str(request.flags) if request.flags else '' - response_flags = ' %s' % str(response.flags) if response.flags else '' + request_flags = f' {str(request.flags)}' if request.flags else '' + response_flags = f' {str(response.flags)}' if response.flags else '' return { 'level': logging.DEBUG, 'msg': CRAWLEDMSG, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 5a2184681..99a72aa70 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -108,7 +108,7 @@ class S3FilesStore: from boto.s3.connection import S3Connection self.S3Connection = S3Connection if not uri.startswith("s3://"): - raise ValueError("Incorrect URI scheme in %s, expected 's3'" % uri) + raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") self.bucket, self.prefix = uri[5:].split('/', 1) def stat_file(self, path, info): @@ -133,7 +133,7 @@ class S3FilesStore: return c.get_bucket(self.bucket, validate=False) def _get_boto_key(self, path): - key_name = '%s%s' % (self.prefix, path) + key_name = f'{self.prefix}{path}' if self.is_botocore: return threads.deferToThread( self.s3_client.head_object, @@ -145,7 +145,7 @@ class S3FilesStore: def persist_file(self, path, buf, info, meta=None, headers=None): """Upload file to S3 storage""" - key_name = '%s%s' % (self.prefix, path) + key_name = f'{self.prefix}{path}' buf.seek(0) if self.is_botocore: extra = self._headers_to_botocore_kwargs(self.HEADERS) @@ -208,8 +208,7 @@ class S3FilesStore: try: kwarg = mapping[key] except KeyError: - raise TypeError( - 'Header "%s" is not supported by botocore' % key) + raise TypeError(f'Header "{key}" is not supported by botocore') else: extra[kwarg] = value return extra @@ -283,7 +282,7 @@ class FTPFilesStore: def __init__(self, uri): if not uri.startswith("ftp://"): - raise ValueError("Incorrect URI scheme in %s, expected 'ftp'" % uri) + raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'") u = urlparse(uri) self.port = u.port self.host = u.hostname @@ -293,7 +292,7 @@ class FTPFilesStore: self.basedir = u.path.rstrip('/') def persist_file(self, path, buf, info, meta=None, headers=None): - path = '%s/%s' % (self.basedir, path) + path = f'{self.basedir}/{path}' return threads.deferToThread( ftp_store_file, path=path, file=buf, host=self.host, port=self.port, username=self.username, @@ -308,10 +307,10 @@ class FTPFilesStore: ftp.login(self.username, self.password) if self.USE_ACTIVE_MODE: ftp.set_pasv(False) - file_path = "%s/%s" % (self.basedir, path) - last_modified = float(ftp.voidcmd("MDTM %s" % file_path)[4:].strip()) + file_path = f"{self.basedir}/{path}" + last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) m = hashlib.md5() - ftp.retrbinary('RETR %s' % file_path, m.update) + ftp.retrbinary(f'RETR {file_path}', m.update) return {'last_modified': last_modified, 'checksum': m.hexdigest()} # The file doesn't exist except Exception: @@ -515,7 +514,7 @@ class FilesPipeline(MediaPipeline): def inc_stats(self, spider, status): spider.crawler.stats.inc_value('file_count', spider=spider) - spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider) + spider.crawler.stats.inc_value(f'file_status_count/{status}', spider=spider) # Overridable Interface def get_media_requests(self, item, info): @@ -545,4 +544,4 @@ class FilesPipeline(MediaPipeline): media_type = mimetypes.guess_type(request.url)[0] if media_type: media_ext = mimetypes.guess_extension(media_type) - return 'full/%s%s' % (media_guid, media_ext) + return f'full/{media_guid}{media_ext}' diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 0a67a0b1d..aafd1d8b2 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -125,8 +125,9 @@ class ImagesPipeline(FilesPipeline): width, height = orig_image.size if width < self.min_width or height < self.min_height: - raise ImageException("Image too small (%dx%d < %dx%d)" % - (width, height, self.min_width, self.min_height)) + raise ImageException("Image too small " + f"({width}x{height} < " + f"{self.min_width}x{self.min_height})") image, buf = self.convert_image(orig_image) yield path, image, buf @@ -168,8 +169,8 @@ class ImagesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - return 'full/%s.jpg' % (image_guid) + return f'full/{image_guid}.jpg' def thumb_path(self, request, thumb_id, response=None, info=None): thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid) + return f'thumbs/{thumb_id}/{thumb_guid}.jpg' diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 2439de9a5..0a12f3e2c 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -61,7 +61,7 @@ class MediaPipeline: 'MYPIPE_IMAGES' """ class_name = self.__class__.__name__ - formatted_key = "{}_{}".format(class_name.upper(), key) + formatted_key = f"{class_name.upper()}_{key}" if ( not base_class_name or class_name == base_class_name @@ -151,9 +151,8 @@ class MediaPipeline: if 'item' not in sig.parameters: old_params = str(sig)[1:-1] new_params = old_params + ", *, item=None" - warn('%s(self, %s) is deprecated, ' - 'please use %s(self, %s)' - % (func.__name__, old_params, func.__name__, new_params), + warn(f'{func.__name__}(self, {old_params}) is deprecated, ' + f'please use {func.__name__}(self, {new_params})', ScrapyDeprecationWarning, stacklevel=2) self._expects_item[func.__name__] = False diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index e13d389ee..a9aa6c649 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -141,17 +141,16 @@ class DownloaderAwarePriorityQueue: def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0: - raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP' - % (self.__class__,)) + raise ValueError(f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP') if slot_startprios and not isinstance(slot_startprios, dict): raise ValueError("DownloaderAwarePriorityQueue accepts " - "``slot_startprios`` as a dict; %r instance " + "``slot_startprios`` as a dict; " + f"{slot_startprios.__class__!r} instance " "is passed. Most likely, it means the state is" "created by an incompatible priority queue. " "Only a crawl started with the same priority " - "queue class can be resumed." % - slot_startprios.__class__) + "queue class can be resumed.") self._downloader_interface = DownloaderInterface(crawler) self.downstream_queue_cls = downstream_queue_cls diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index d207088e6..6ed9f8b8f 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -45,7 +45,7 @@ class ResponseTypes: elif mimetype in self.classes: return self.classes[mimetype] else: - basetype = "%s/*" % mimetype.split('/')[0] + basetype = f"{mimetype.split('/')[0]}/*" return self.classes.get(basetype, Response) def from_content_type(self, content_type, content_encoding=None): diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index f12c61081..a25871433 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -66,8 +66,8 @@ class Selector(_ParselSelector, object_ref): def __init__(self, response=None, text=None, type=None, root=None, **kwargs): if response is not None and text is not None: - raise ValueError('%s.__init__() received both response and text' - % self.__class__.__name__) + raise ValueError(f'{self.__class__.__name__}.__init__() received ' + 'both response and text') st = _st(response, type or self._default_type) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 951fc65e2..1fe1e6fd1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -52,7 +52,7 @@ class SettingsAttribute: self.priority = priority def __str__(self): - return "".format(self=self) + return f"" __repr__ = __str__ diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a0251394b..4ef330dd2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -287,7 +287,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) URLLENGTH_LIMIT = 2083 -USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__version__ +USER_AGENT = f'Scrapy/{import_module("scrapy").__version__} (+https://scrapy.org)' TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] diff --git a/scrapy/shell.py b/scrapy/shell.py index 10de119ce..c370ccaff 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -140,7 +140,7 @@ class Shell: b.append(" scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)") for k, v in sorted(self.vars.items()): if self._is_relevant(v): - b.append(" %-10s %s" % (k, v)) + b.append(f" {k:<10} {v}") b.append("Useful shortcuts:") if self.inthread: b.append(" fetch(url[, redirect=True]) " @@ -150,7 +150,7 @@ class Shell: b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") - return "\n".join("[s] %s" % line for line in b) + return "\n".join(f"[s] {line}" for line in b) def _is_relevant(self, value): return isinstance(value, self.relevant_classes) or is_item(value) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index db4193430..04fda311f 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -27,7 +27,7 @@ class SpiderLoader: dupes = [] for name, locations in self._found.items(): dupes.extend([ - " {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name) + f" {cls} named {name!r} (in {mod})" for mod, cls in locations if len(locations) > 1 ]) @@ -36,7 +36,7 @@ class SpiderLoader: dupes_string = "\n\n".join(dupes) warnings.warn( "There are several spiders with the same name:\n\n" - "{}\n\n This can cause unexpected behavior.".format(dupes_string), + f"{dupes_string}\n\n This can cause unexpected behavior.", category=UserWarning, ) @@ -53,10 +53,9 @@ class SpiderLoader: except ImportError: if self.warn_only: warnings.warn( - "\n{tb}Could not load spiders from module '{modname}'. " - "See above traceback for details.".format( - modname=name, tb=traceback.format_exc() - ), + f"\n{traceback.format_exc()}Could not load spiders " + f"from module '{name}'. " + "See above traceback for details.", category=RuntimeWarning, ) else: @@ -75,7 +74,7 @@ class SpiderLoader: try: return self._spiders[spider_name] except KeyError: - raise KeyError("Spider not found: {}".format(spider_name)) + raise KeyError(f"Spider not found: {spider_name}") def find_by_request(self, request): """ diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index fa7f5bef9..776a6879a 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -43,7 +43,7 @@ class DepthMiddleware: return False else: if self.verbose_stats: - self.stats.inc_value('request_depth_count/%s' % depth, + self.stats.inc_value(f'request_depth_count/{depth}', spider=spider) self.stats.max_value('request_depth_max', depth, spider=spider) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index db9d0f2ae..ae5c258df 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -48,7 +48,7 @@ class HttpErrorMiddleware: if isinstance(exception, HttpError): spider.crawler.stats.inc_value('httperror/response_ignored_count') spider.crawler.stats.inc_value( - 'httperror/response_ignored_status_count/%s' % response.status + f'httperror/response_ignored_status_count/{response.status}' ) logger.info( "Ignoring response %(response)r: HTTP status code is not handled or not allowed", diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index a006f3177..6e4efda97 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -61,15 +61,15 @@ class OffsiteMiddleware: continue elif url_pattern.match(domain): message = ("allowed_domains accepts only domains, not URLs. " - "Ignoring URL entry %s in allowed_domains." % domain) + f"Ignoring URL entry {domain} in allowed_domains.") warnings.warn(message, URLWarning) elif port_pattern.search(domain): message = ("allowed_domains accepts only domains without ports. " - "Ignoring entry %s in allowed_domains." % domain) + f"Ignoring entry {domain} in allowed_domains.") warnings.warn(message, PortWarning) else: domains.append(re.escape(domain)) - regex = r'^(.*\.)?(%s)$' % '|'.join(domains) + regex = fr'^(.*\.)?({"|".join(domains)})$' return re.compile(regex) def spider_opened(self, spider): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 434067b00..f81041376 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -278,7 +278,7 @@ def _load_policy_class(policy, warning_only=False): try: return _policy_classes[policy.lower()] except KeyError: - msg = "Could not load referrer policy %r" % policy + msg = f"Could not load referrer policy {policy!r}" if not warning_only: raise RuntimeError(msg) else: diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 12b4fba09..3da0a11db 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -25,7 +25,7 @@ class Spider(object_ref): if name is not None: self.name = name elif not getattr(self, 'name', None): - raise ValueError("%s must have a name" % type(self).__name__) + raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) if not hasattr(self, 'start_urls'): self.start_urls = [] @@ -66,9 +66,8 @@ class Spider(object_ref): warnings.warn( "Spider.make_requests_from_url method is deprecated; it " "won't be called in future Scrapy releases. Please " - "override Spider.start_requests method instead (see %s.%s)." % ( - cls.__module__, cls.__name__ - ), + "override Spider.start_requests method instead " + f"(see {cls.__module__}.{cls.__name__}).", ) for url in self.start_urls: yield self.make_requests_from_url(url) @@ -90,7 +89,7 @@ class Spider(object_ref): return self.parse(response, **kwargs) def parse(self, response, **kwargs): - raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) + raise NotImplementedError(f'{self.__class__.__name__}.parse callback is not defined') @classmethod def update_settings(cls, settings): @@ -107,7 +106,7 @@ class Spider(object_ref): return closed(reason) def __str__(self): - return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self)) + return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>" __repr__ = __str__ diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index cf658aec4..6ed17e4dd 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -71,11 +71,11 @@ class XMLFeedSpider(Spider): elif self.iterator == 'xml': selector = Selector(response, type='xml') self._register_namespaces(selector) - nodes = selector.xpath('//%s' % self.itertag) + nodes = selector.xpath(f'//{self.itertag}') elif self.iterator == 'html': selector = Selector(response, type='html') self._register_namespaces(selector) - nodes = selector.xpath('//%s' % self.itertag) + nodes = selector.xpath(f'//{self.itertag}') else: raise NotSupported('Unsupported node iterator') diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index f595a1acb..86238c4cd 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -21,8 +21,8 @@ class Root(Resource): for nl in nlist: args['n'] = nl argstr = urlencode(args, doseq=True) - request.write("follow {1}
    " - .format(argstr, nl).encode('utf8')) + request.write(f"follow {nl}
    " + .encode('utf8')) request.write(b"") return b'' @@ -39,6 +39,6 @@ if __name__ == '__main__': def _print_listening(): httpHost = httpPort.getHost() - print("Bench server at http://{}:{}".format(httpHost.host, httpHost.port)) + print(f"Bench server at http://{httpHost.host}:{httpHost.port}") reactor.callWhenRunning(_print_listening) reactor.run() diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 90a52b25b..05cd5f25c 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -17,8 +17,8 @@ def build_component_list(compdict, custom=None, convert=update_classpath): def _check_components(complist): if len({convert(c) for c in complist}) != len(complist): - raise ValueError('Some paths in {!r} convert to the same object, ' - 'please update your settings'.format(complist)) + raise ValueError('Some paths in {complist!r} convert to the same object, ' + 'please update your settings') def _map_keys(compdict): if isinstance(compdict, BaseSettings): @@ -26,9 +26,10 @@ def build_component_list(compdict, custom=None, convert=update_classpath): for k, v in compdict.items(): prio = compdict.getpriority(k) if compbs.getpriority(convert(k)) == prio: - raise ValueError('Some paths in {!r} convert to the same ' + raise ValueError(f'Some paths in {list(compdict.keys())!r} ' + 'convert to the same ' 'object, please update your settings' - ''.format(list(compdict.keys()))) + ) else: compbs.set(convert(k), v, priority=prio) return compbs @@ -40,8 +41,9 @@ def build_component_list(compdict, custom=None, convert=update_classpath): """Fail if a value in the components dict is not a real number or None.""" for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): - raise ValueError('Invalid value {} for component {}, please provide ' - 'a real number or None instead'.format(value, name)) + raise ValueError(f'Invalid value {value} for component {name}, ' + 'please provide a real number or None instead' + ) # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): @@ -141,12 +143,10 @@ def feed_process_params_from_cli(settings, output, output_format=None, def check_valid_format(output_format): if output_format not in valid_output_formats: raise UsageError( - "Unrecognized output format '%s'. Set a supported one (%s) " + f"Unrecognized output format '{output_format}'. " + f"Set a supported one ({tuple(valid_output_formats)}) " "after a colon at the end of the output URI (i.e. -o/-O " - ":) or as a file extension." % ( - output_format, - tuple(valid_output_formats), - ) + ":) or as a file extension." ) overwrite = False diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 9c0efcec4..6660b9dc0 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -9,7 +9,7 @@ from w3lib.http import basic_auth_header class CurlParser(argparse.ArgumentParser): def error(self, message): - error_msg = 'There was an error parsing the curl command: {}'.format(message) + error_msg = f'There was an error parsing the curl command: {message}' raise ValueError(error_msg) @@ -52,7 +52,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): parsed_args, argv = curl_parser.parse_known_args(curl_args[1:]) if argv: - msg = 'Unrecognized options: {}'.format(', '.join(argv)) + msg = f'Unrecognized options: {", ".join(argv)}' if ignore_unknown_options: warnings.warn(msg) else: diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 2e2c7adc1..fef3882cb 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -14,9 +14,9 @@ def deprecated(use_instead=None): def deco(func): @wraps(func) def wrapped(*args, **kwargs): - message = "Call to deprecated function %s." % func.__name__ + message = f"Call to deprecated function {func.__name__}." if use_instead: - message += " Use %s instead." % use_instead + message += f" Use {use_instead} instead." warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) return func(*args, **kwargs) return wrapped diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3c8e3c8b5..fb7e69889 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -8,9 +8,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning def attribute(obj, oldattr, newattr, version='0.12'): cname = obj.__class__.__name__ warnings.warn( - "%s.%s attribute is deprecated and will be no longer supported " - "in Scrapy %s, use %s.%s attribute instead" - % (cname, oldattr, version, cname, newattr), + f"{cname}.{oldattr} attribute is deprecated and will be no longer supported " + f"in Scrapy {version}, use {cname}.{newattr} attribute instead", ScrapyDeprecationWarning, stacklevel=3) @@ -116,7 +115,7 @@ def create_deprecated_class( # deprecated class is in jinja2 template). __module__ attribute is not # important enough to raise an exception as users may be unable # to fix inspect.stack() errors. - warnings.warn("Error detecting parent module: %r" % e) + warnings.warn(f"Error detecting parent module: {e!r}") return deprecated_cls @@ -124,7 +123,7 @@ def create_deprecated_class( def _clspath(cls, forced=None): if forced is not None: return forced - return '{}.{}'.format(cls.__module__, cls.__name__) + return f'{cls.__module__}.{cls.__name__}' DEPRECATION_RULES = [ @@ -137,7 +136,7 @@ def update_classpath(path): for prefix, replacement in DEPRECATION_RULES: if path.startswith(prefix): new_path = path.replace(prefix, replacement, 1) - warnings.warn("`{}` class is deprecated, use `{}` instead".format(path, new_path), + warnings.warn(f"`{path}` class is deprecated, use `{new_path}` instead", ScrapyDeprecationWarning) return new_path return path diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 267c7ecd1..0c1cee1a0 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -29,7 +29,7 @@ def get_engine_status(engine): try: checks += [(test, eval(test))] except Exception as e: - checks += [(test, "%s (exception)" % type(e).__name__)] + checks += [(test, f"{type(e).__name__} (exception)")] return checks @@ -38,7 +38,7 @@ def format_engine_status(engine=None): checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: - s += "%-47s : %s\n" % (test, result) + s += f"{test:<47} : {result}\n" s += "\n" return s diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 19d56d6ec..6cace4f07 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -33,5 +33,5 @@ def ftp_store_file( dirname, filename = posixpath.split(path) ftp_makedirs_cwd(ftp, dirname) command = 'STOR' if overwrite else 'APPE' - ftp.storbinary('%s %s' % (command, filename), file) + ftp.storbinary(f'{command} {filename}', file) file.close() diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 5e15bf0c8..789da1392 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -22,8 +22,8 @@ def xmliter(obj, nodename): """ nodename_patt = re.escape(nodename) - HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename_patt, re.S) - HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename_patt, re.S) + HEADER_START_RE = re.compile(fr'^(.*?)<\s*{nodename_patt}(?:\s|>)', re.S) + HEADER_END_RE = re.compile(fr'<\s*/{nodename_patt}\s*>', re.S) text = _body_or_str(obj) header_start = re.search(HEADER_START_RE, text) @@ -31,7 +31,7 @@ def xmliter(obj, nodename): header_end = re_rsearch(HEADER_END_RE, text) header_end = text[header_end[1]:].strip() if header_end else '' - r = re.compile(r'<%(np)s[\s>].*?' % {'np': nodename_patt}, re.DOTALL) + r = re.compile(fr'<{nodename_patt}[\s>].*?', re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] @@ -40,9 +40,9 @@ def xmliter(obj, nodename): def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): from lxml import etree reader = _StreamReader(obj) - tag = '{%s}%s' % (namespace, nodename) if namespace else nodename + tag = f'{{{namespace}}}{nodename}'if namespace else nodename iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) - selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) + selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename) for _, node in iterable: nodetext = etree.tostring(node, encoding='unicode') node.clear() @@ -131,8 +131,7 @@ def _body_or_str(obj, unicode=True): if not isinstance(obj, expected_types): expected_types_str = " or ".join(t.__name__ for t in expected_types) raise TypeError( - "Object %r must be %s, not %s" - % (obj, expected_types_str, type(obj).__name__) + f"Object {obj!r} must be {expected_types_str}, not {type(obj).__name__}" ) if isinstance(obj, Response): if not unicode: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index e41315738..62df7a6ab 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -143,7 +143,7 @@ def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) versions = [ - "%s %s" % (name, version) + f"{name} {version}" for name, version in scrapy_components_versions() if name != "Scrapy" ] @@ -187,7 +187,7 @@ class LogCounterHandler(logging.Handler): self.crawler = crawler def emit(self, record): - sname = 'log_count/{}'.format(record.levelname) + sname = f'log_count/{record.levelname}' self.crawler.stats.inc_value(sname) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index bd400bd30..9107f30ef 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -46,7 +46,7 @@ def load_object(path): try: dot = path.rindex('.') except ValueError: - raise ValueError("Error loading object '%s': not a full path" % path) + raise ValueError(f"Error loading object '{path}': not a full path") module, name = path[:dot], path[dot + 1:] mod = import_module(module) @@ -54,7 +54,7 @@ def load_object(path): try: obj = getattr(mod, name) except AttributeError: - raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name)) + raise NameError(f"Module '{module}' doesn't define any object named '{name}'") return obj @@ -163,7 +163,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): instance = objcls(*args, **kwargs) method_name = '__new__' if instance is None: - raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name)) + raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") return instance @@ -234,9 +234,10 @@ def warn_on_generator_with_return_value(spider, callable): """ if is_generator_with_return_value(callable): warnings.warn( - 'The "{}.{}" method is a generator and includes a "return" statement with a ' - 'value different than None. This could lead to unexpected behaviour. Please see ' + f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' + 'a generator and includes a "return" statement with a value ' + 'different than None. This could lead to unexpected behaviour. Please see ' 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' - 'for details about the semantics of the "return" statement within generators' - .format(spider.__class__.__name__, callable.__name__), stacklevel=2, + 'for details about the semantics of the "return" statement within generators', + stacklevel=2, ) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index b8d3ebf9d..fd13d85e3 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -20,7 +20,7 @@ def inside_project(): try: import_module(scrapy_module) except ImportError as exc: - warnings.warn("Cannot import scrapy settings module %s: %s" % (scrapy_module, exc)) + warnings.warn(f"Cannot import scrapy settings module {scrapy_module}: {exc}") else: return True return bool(closest_scrapy_cfg()) @@ -90,7 +90,7 @@ def get_project_settings(): warnings.warn( 'Use of environment variables prefixed with SCRAPY_ to override ' 'settings is deprecated. The following environment variables are ' - 'currently defined: {}'.format(setting_envvar_list), + f'currently defined: {setting_envvar_list}', ScrapyDeprecationWarning ) settings.setdict(scrapy_envvars, priority='project') diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 1f2333264..5703fd4c3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -91,7 +91,7 @@ def to_unicode(text, encoding=None, errors='strict'): return text if not isinstance(text, (bytes, str)): raise TypeError('to_unicode must receive a bytes or str ' - 'object, got %s' % type(text).__name__) + f'object, got {type(text).__name__}') if encoding is None: encoding = 'utf-8' return text.decode(encoding, errors) @@ -104,7 +104,7 @@ def to_bytes(text, encoding=None, errors='strict'): return text if not isinstance(text, str): raise TypeError('to_bytes must receive a str or bytes ' - 'object, got %s' % type(text).__name__) + f'object, got {type(text).__name__}') if encoding is None: encoding = 'utf-8' return text.encode(encoding, errors) @@ -174,7 +174,7 @@ def binary_is_text(data): does not contain unprintable control characters. """ if not isinstance(data, bytes): - raise TypeError("data must be bytes, got '%s'" % type(data).__name__) + raise TypeError(f"data must be bytes, got '{type(data).__name__}'") return all(c not in _BINARYCHARS for c in data) @@ -217,7 +217,7 @@ def get_func_args(func, stripself=False): else: return get_func_args(func.__call__, True) else: - raise TypeError('%s is not callable' % type(func)) + raise TypeError(f'{type(func)} is not callable') if stripself: func_args.pop(0) return func_args @@ -250,7 +250,7 @@ def get_spec(func): elif hasattr(func, '__call__'): spec = _getargspec_py23(func.__call__) else: - raise TypeError('%s is not callable' % type(func)) + raise TypeError(f'{type(func)} is not callable') defaults = spec.defaults or [] @@ -322,7 +322,7 @@ def global_object_name(obj): >>> global_object_name(Request) 'scrapy.http.request.Request' """ - return "%s.%s" % (obj.__module__, obj.__name__) + return f"{obj.__module__}.{obj.__name__}" if hasattr(sys, "pypy_version_info"): diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 879d27907..831d29462 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -10,7 +10,7 @@ def listen_tcp(portrange, host, factory): """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor if len(portrange) > 2: - raise ValueError("invalid portrange: %s" % portrange) + raise ValueError(f"invalid portrange: {portrange}") if not portrange: return reactor.listenTCP(0, factory, interface=host) if not hasattr(portrange, '__iter__'): @@ -78,9 +78,9 @@ def verify_installed_reactor(reactor_path): from twisted.internet import reactor reactor_class = load_object(reactor_path) if not isinstance(reactor, reactor_class): - msg = "The installed reactor ({}.{}) does not match the requested one ({})".format( - reactor.__module__, reactor.__class__.__name__, reactor_path - ) + msg = ("The installed reactor " + f"({reactor.__module__}.{reactor.__class__.__name__}) does not " + f"match the requested one ({reactor_path})") raise Exception(msg) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 503d7b133..d38b1bc4d 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -84,7 +84,7 @@ def _find_method(obj, func): # https://docs.python.org/3/reference/datamodel.html if obj_func.__func__ is func.__func__: return name - raise ValueError("Function %s is not an instance method in: %s" % (func, obj)) + raise ValueError(f"Function {func} is not an instance method in: {obj}") def _get_method(obj, name): @@ -92,4 +92,4 @@ def _get_method(obj, name): try: return getattr(obj, name) except AttributeError: - raise ValueError("Method %r not found in: %s" % (name, obj)) + raise ValueError(f"Method {name!r} not found in: {obj}") diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c29b619ce..99b089b6f 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -39,7 +39,7 @@ def response_status_message(status): """Return status code plus status text descriptive message """ message = http.RESPONSES.get(int(status), "Unknown Status") - return '%s %s' % (status, to_unicode(message)) + return f'{status} {to_unicode(message)}' def response_httprepr(response): @@ -69,15 +69,15 @@ def open_in_browser(response, _openfunc=webbrowser.open): body = response.body if isinstance(response, HtmlResponse): if b'' body = body.replace(b'', to_bytes(repl)) ext = '.html' elif isinstance(response, TextResponse): ext = '.txt' else: - raise TypeError("Unsupported response type: %s" % - response.__class__.__name__) + raise TypeError("Unsupported response type: " + f"{response.__class__.__name__}") fd, fname = tempfile.mkstemp(ext) os.write(fd, body) os.close(fd) - return _openfunc("file://%s" % fname) + return _openfunc(f"file://{fname}") diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index cc3263602..a73cf03c5 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -17,7 +17,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): if isinstance(o, set): return list(o) elif isinstance(o, datetime.datetime): - return o.strftime("%s %s" % (self.DATE_FORMAT, self.TIME_FORMAT)) + return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}") elif isinstance(o, datetime.date): return o.strftime(self.DATE_FORMAT) elif isinstance(o, datetime.time): @@ -29,9 +29,9 @@ class ScrapyJSONEncoder(json.JSONEncoder): elif is_item(o): return ItemAdapter(o).asdict() elif isinstance(o, Request): - return "<%s %s %s>" % (type(o).__name__, o.method, o.url) + return f"<{type(o).__name__} {o.method} {o.url}>" elif isinstance(o, Response): - return "<%s %s %s>" % (type(o).__name__, o.status, o.url) + return f"<{type(o).__name__} {o.status} {o.url}>" else: return super().default(o) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index c3c5e329b..ea4dde882 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -50,7 +50,7 @@ def get_temp_key_info(ssl_object): key_info.append(ffi_buf_to_string(cname)) else: key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) - key_info.append('%s bits' % pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)) + key_info.append(f'{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits') return ', '.join(key_info) @@ -58,4 +58,4 @@ def get_openssl_version(): system_openssl = OpenSSL.SSL.SSLeay_version( OpenSSL.SSL.SSLEAY_VERSION ).decode('ascii', errors='replace') - return '{} ({})'.format(OpenSSL.version.__version__, system_openssl) + return f'{OpenSSL.version.__version__} ({system_openssl})' diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 7442a2f33..f54942ffb 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -79,7 +79,7 @@ def get_ftp_content_and_delete( def buffer_data(data): ftp_data.append(data) - ftp.retrbinary('RETR %s' % path, buffer_data) + ftp.retrbinary(f'RETR {path}', buffer_data) dirname, filename = split(path) ftp.cwd(dirname) ftp.delete(filename) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index a63c9a942..a54c7db95 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -23,10 +23,10 @@ class ProcessTest: def _process_finished(self, pp, cmd, check_code): if pp.exitcode and check_code: - msg = "process %s exit with code %d" % (cmd, pp.exitcode) - msg += "\n>>> stdout <<<\n%s" % pp.out + msg = f"process {cmd} exit with code {pp.exitcode}" + msg += f"\n>>> stdout <<<\n{pp.out}" msg += "\n" - msg += "\n>>> stderr <<<\n%s" % pp.err + msg += f"\n>>> stderr <<<\n{pp.err}" raise RuntimeError(msg) return pp.exitcode, pp.out, pp.err diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 397e54703..fce77be32 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -9,7 +9,7 @@ class SiteTest: from twisted.internet import reactor super().setUp() self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - self.baseurl = "http://localhost:%d/" % self.site.getHost().port + self.baseurl = f"http://localhost:{self.site.getHost().port}/" def tearDown(self): super().tearDown() @@ -40,5 +40,5 @@ def test_site(): if __name__ == '__main__': from twisted.internet import reactor port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - print("http://localhost:%d/" % port.getHost().port) + print(f"http://localhost:{port.getHost().port}/") reactor.run() diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index baed5c536..3e40acd69 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -41,9 +41,7 @@ def format_live_refs(ignore=NoneType): if issubclass(cls, ignore): continue oldest = min(wdict.values()) - s += "%-30s %6d oldest: %ds ago\n" % ( - cls.__name__, len(wdict), now - oldest - ) + s += f"{cls.__name__:<30} {len(wdict):6} oldest: {int(now - oldest)}s ago\n" return s diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index b23ddb459..a6a2a9e8b 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -22,7 +22,7 @@ def url_is_from_any_domain(url, domains): if not host: return False domains = [d.lower() for d in domains] - return any((host == d) or (host.endswith('.%s' % d)) for d in domains) + return any((host == d) or (host.endswith(f'.{d}')) for d in domains) def url_is_from_spider(url, spider): @@ -153,7 +153,7 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only= if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443), ('ftp', 21)): - netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') + netloc = netloc.replace(f':{parsed_url.port}', '') return urlunparse(( parsed_url.scheme, netloc, diff --git a/sep/sep-002.rst b/sep/sep-002.rst index c467cb402..2e8a28340 100644 --- a/sep/sep-002.rst +++ b/sep/sep-002.rst @@ -30,7 +30,7 @@ Proposed Implementation if hasattr(value, '__iter__'): # str/unicode not allowed return [self._field.to_python(v) for v in value] else: - raise TypeError("Expected iterable, got %s" % type(value).__name__) + raise TypeError(f"Expected iterable, got {type(value).__name__}") def get_default(self): # must return a new copy to avoid unexpected behaviors with mutable defaults diff --git a/sep/sep-004.rst b/sep/sep-004.rst index 05b0eb99c..b9f5e556f 100644 --- a/sep/sep-004.rst +++ b/sep/sep-004.rst @@ -11,7 +11,7 @@ SEP-004: Library API ==================== .. note:: the library API has been implemented, but slightly different from proposed in this SEP. You can run a Scrapy crawler inside a Twisted - reactor, but not outside it. + reactor, but not outside it. Introduction ============ @@ -49,7 +49,7 @@ Here's a simple proof-of-concept code of such script: cr = Crawler(start_urls, callback=parse_start_page) cr.run() # blocking call - this populates scraped_items - print "%d items scraped" % len(scraped_items) + print(f"{len(scraped_items)} items scraped") # ... do something more interesting with scraped_items ... The behaviour of the Scrapy crawler would be controller by the Scrapy settings, diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 8ca81824d..4e3340521 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -21,7 +21,7 @@ Current flaws and inconsistencies 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) 3. Isn't possible to crawl an url directly from command line because the Spider - does not know which callback use. + does not know which callback use. These flaws will be corrected by the changes proposed in this SEP. @@ -55,7 +55,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. This is an enhancement to ``LinkExtractors`` which returns urls (links), -Request Extractors return Request objects. +Request Extractors return Request objects. Request Processors ------------------ @@ -142,7 +142,7 @@ Custom Processor and External Callback # Callback defined out of spider def my_external_callback(response): - # process item + # process item pass class SampleSpider(CrawlSpider): @@ -233,7 +233,7 @@ Request/Response Matchers def matches_request(self, request): """Returns True if Request's url matches initial url""" - return self.matches_url(request.url) + return self.matches_url(request.url) def matches_response(self, response): """REturns True if Response's url matches initial url""" @@ -305,14 +305,14 @@ Request Extractor for req in self.requests: req.meta.setdefault('link_text', '') req.meta['link_text'] = str_to_unicode(req.meta['link_text'], - encoding) + encoding) def reset(self): """Reset state""" FixedSGMLParser.reset(self) self.requests = [] self.base_url = None - + def unknown_starttag(self, tag, attrs): """Process unknown start tag""" if 'base' tag: @@ -376,7 +376,7 @@ Request Processor #!python # - # Request Processors + # Request Processors # Processors receive list of requests and return list of requests # """Request Processors""" @@ -390,7 +390,7 @@ Request Processor # replace in-place req.url = canonicalize_url(req.url) yield req - + class Unique(object): """Filter duplicate Requests""" @@ -455,9 +455,9 @@ Request Processor """Initialize allow/deny attributes""" _re_type = type(re.compile('', 0)) - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] def __call__(self, requests): @@ -524,7 +524,7 @@ Rules Manager # # Handles rules matcher/callbacks # Resolve rule for given response - # + # class RulesManager(object): """Rules Manager""" def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): @@ -542,8 +542,8 @@ Rules Manager # instance default matcher matcher = default_matcher(rule.matcher) else: - raise ValueError('Not valid matcher given %r in %r' \ - % (rule.matcher, rule)) + raise ValueError('Not valid matcher given ' + f'{rule.matcher!r} in {rule!r}') # prepare callback if callable(rule.callback): @@ -553,8 +553,7 @@ Rules Manager callback = getattr(spider, rule.callback) if not callable(callback): - raise AttributeError('Invalid callback %r can not be resolved' \ - % callback) + raise AttributeError(f'Invalid callback {callback!r} can not be resolved') else: callback = None diff --git a/sep/sep-018.rst b/sep/sep-018.rst index fe707923a..d0169b81e 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -171,7 +171,7 @@ the same spider: #!python class MySpider(BaseSpider): - middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), + middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), ItemIdSetter(), OffsiteMiddleware()] allowed_domains = ['example.com', 'sub.example.com'] @@ -196,7 +196,7 @@ the same spider: # extract item from response return item -The Spider Middleware that implements spider code +The Spider Middleware that implements spider code ================================================= There's gonna be one middleware that will take care of calling the proper @@ -324,7 +324,7 @@ Another example could be for building URL canonicalizers: class CanonializeUrl(object): def process_request(self, request, response, spider): - curl = canonicalize_url(request.url, + curl = canonicalize_url(request.url, rules=spider.canonicalization_rules) return request.replace(url=curl) @@ -332,7 +332,7 @@ Another example could be for building URL canonicalizers: class MySpider(BaseSpider): middlewares = [CanonicalizeUrl()] - canonicalization_rules = ['sort-query-args', + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] # ... @@ -414,7 +414,7 @@ A spider middleware to avoid visiting pages forbidden by robots.txt: if netloc in info.pending: res = None else: - robotsurl = "%s://%s/robots.txt" % (url.scheme, netloc) + robotsurl = f"{url.scheme}://{netloc}/robots.txt" meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]} res = Request(robotsurl, callback=self.parse_robots, meta=meta, priority=self.REQUEST_PRIORITY) @@ -474,7 +474,7 @@ This is a port of the Offsite middleware to the new spider middleware API: if host and host not in info.hosts_seen: spider.log("Filtered offsite request to %r: %s" % (host, request)) info.hosts_seen.add(host) - + def should_follow(self, request, spider): info = self.spiders[spider] # hostname can be None for wrong urls (like javascript links) @@ -484,7 +484,7 @@ This is a port of the Offsite middleware to the new spider middleware API: def get_host_regex(self, spider): """Override this method to implement a different offsite policy""" domains = [d.replace('.', r'\.') for d in spider.allowed_domains] - regex = r'^(.*\.)?(%s)$' % '|'.join(domains) + regex = fr'^(.*\.)?({"|".join(domains)})$' return re.compile(regex) def spider_opened(self, spider): @@ -570,7 +570,7 @@ A middleware to filter out requests already seen: self.dupefilter = load_object(clspath)() dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - + def enqueue_request(self, spider, request): seen = self.dupefilter.request_seen(spider, request) if not seen or request.dont_filter: @@ -601,8 +601,8 @@ A middleware to Scrape data using Parsley as described in UsingParsley for name in parslet.keys(): self.fields[name] = Field() super(ParsleyItem, self).__init__(*a, **kw) - self.item_class = ParsleyItem - self.parsley = PyParsley(parslet, output='python') + self.item_class = ParsleyItem + self.parsley = PyParsley(parslet, output='python') def process_response(self, response, request, spider): return self.item_class(self.parsly.parse(string=response.body)) @@ -627,7 +627,7 @@ Resolved: not the original one (think of redirections), but it does carry the ``meta`` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be - the deserialized request from the persistent scheduler queue. + the deserialized request from the persistent scheduler queue. - No - this would make implementation more complex and we're not sure it's really needed diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 3f9738798..f545de39f 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -38,7 +38,7 @@ class LocalhostSpider(Spider): if __name__ == "__main__": with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: port = urlparse(mock_http_server.http_address).port - url = "http://not.a.real.domain:{port}/echo".format(port=port) + url = f"http://not.a.real.domain:{port}/echo" servers = [(mock_dns_server.host, mock_dns_server.port)] reactor.installResolver(createResolver(servers=servers)) diff --git a/tests/mockserver.py b/tests/mockserver.py index 6f0c274b9..ab9aec6a6 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -73,7 +73,7 @@ class Follow(LeafResource): for nl in nlist: args[b"n"] = [to_bytes(str(nl))] argstr = urlencode(args, doseq=True) - s += "follow %d
    " % (argstr, nl) + s += f"follow {nl}
    " s += """""" request.write(to_bytes(s)) request.finish() @@ -91,7 +91,7 @@ class Delay(LeafResource): return NOT_DONE_YET def _delayedRender(self, request, n): - request.write(to_bytes("Response delayed for %0.3f seconds\n" % n)) + request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n")) request.finish() @@ -310,8 +310,8 @@ if __name__ == "__main__": def print_listening(): httpHost = httpPort.getHost() httpsHost = httpsPort.getHost() - httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port) - httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port) + httpAddress = f'http://{httpHost.host}:{httpHost.port}' + httpsAddress = f'https://{httpsHost.host}:{httpsHost.port}' print(httpAddress) print(httpsAddress) @@ -323,7 +323,7 @@ if __name__ == "__main__": def print_listening(): host = listener.getHost() - print("%s:%s" % (host.host, host.port)) + print(f"{host.host}:{host.port}") reactor.callWhenRunning(print_listening) reactor.run() diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py index 162a53760..e49340284 100644 --- a/tests/py36/_test_crawl.py +++ b/tests/py36/_test_crawl.py @@ -33,7 +33,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): depth = 2 def _get_req(self, index, cb=None): - return Request(self.mockserver.url("/status?n=200&request=%d" % index), + return Request(self.mockserver.url(f"/status?n=200&request={index}"), meta={'index': index}, dont_filter=True, callback=cb) diff --git a/tests/spiders.py b/tests/spiders.py index 63bd726fb..3e525e62f 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -45,7 +45,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = self.mockserver.url("/follow?%s" % urlencode(qargs, doseq=1)) + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") self.start_urls = [url] def parse(self, response): @@ -67,7 +67,7 @@ class DelaySpider(MetaSpider): def start_requests(self): self.t1 = time.time() - url = self.mockserver.url("/delay?n=%s&b=%s" % (self.n, self.b)) + url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) def parse(self, response): @@ -192,7 +192,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1) + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 @@ -239,7 +239,7 @@ class DuplicateStartRequestsSpider(MockServerSpider): def start_requests(self): for i in range(0, self.distinct_urls): for j in range(0, self.dupe_factor): - url = self.mockserver.url("/echo?headers=1&body=test%d" % i) + url = self.mockserver.url(f"/echo?headers=1&body=test{i}") yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index 6504b4d2c..005e45214 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -4,7 +4,7 @@ class TestExtension: def __init__(self, settings): - settings.set('TEST1', "%s + %s" % (settings['TEST1'], 'started')) + settings.set('TEST1', f"{settings['TEST1']} + started") @classmethod def from_crawler(cls, crawler): diff --git a/tests/test_command_check.py b/tests/test_command_check.py index f27f526a3..34f5e59dd 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -14,20 +14,20 @@ class CheckCommandTest(CommandTest): def _write_contract(self, contracts, parse_def): with open(self.spider, 'w') as file: - file.write(""" + file.write(f""" import scrapy class CheckSpider(scrapy.Spider): - name = '{0}' + name = '{self.spider_name}' start_urls = ['http://example.com'] def parse(self, response, **cb_kwargs): \"\"\" @url http://example.com - {1} + {contracts} \"\"\" - {2} - """.format(self.spider_name, contracts, parse_def)) + {parse_def} + """) def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index e115f420f..ed3848d88 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -21,14 +21,14 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): self.spider_name = 'parse_spider' fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) with open(fname, 'w') as f: - f.write(""" + f.write(f""" import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule class MySpider(scrapy.Spider): - name = '{0}' + name = '{self.spider_name}' def parse(self, response): if getattr(self, 'test_arg', None): @@ -58,7 +58,7 @@ class MySpider(scrapy.Spider): self.logger.debug('It Does Not Work :(') class MyGoodCrawlSpider(CrawlSpider): - name = 'goodcrawl{0}' + name = 'goodcrawl{self.spider_name}' rules = ( Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), @@ -74,7 +74,7 @@ class MyGoodCrawlSpider(CrawlSpider): class MyBadCrawlSpider(CrawlSpider): '''Spider which doesn't define a parse_item callback while using it in a rule.''' - name = 'badcrawl{0}' + name = 'badcrawl{self.spider_name}' rules = ( Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), @@ -82,7 +82,7 @@ class MyBadCrawlSpider(CrawlSpider): def parse(self, response): return [scrapy.Item(), dict(foo='bar')] -""".format(self.spider_name)) +""") fname = abspath(join(self.proj_mod_path, 'pipelines.py')) with open(fname, 'w') as f: @@ -99,9 +99,9 @@ class MyPipeline: fname = abspath(join(self.proj_mod_path, 'settings.py')) with open(fname, 'a') as f: - f.write(""" -ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} -""" % self.project_name) + f.write(f""" +ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} +""") @defer.inlineCallbacks def test_spider_arguments(self): diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 66c293c00..16c9559b5 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -65,8 +65,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): def test_fetch_redirect_follow_302(self): """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" url = self.url('/redirect-no-meta-refresh') - code = "fetch('{0}')" - errcode, out, errout = yield self.execute(['-c', code.format(url)]) + code = f"fetch('{url}')" + errcode, out, errout = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) assert b'Redirecting (302)' in errout assert b'Crawled (200)' in errout @@ -75,23 +75,23 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): def test_fetch_redirect_not_follow_302(self): """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" url = self.url('/redirect-no-meta-refresh') - code = "fetch('{0}', redirect=False)" - errcode, out, errout = yield self.execute(['-c', code.format(url)]) + code = f"fetch('{url}', redirect=False)" + errcode, out, errout = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) assert b'Crawled (302)' in errout @defer.inlineCallbacks def test_request_replace(self): url = self.url('/text') - code = "fetch('{0}') or fetch(response.request.replace(method='POST'))" - errcode, out, _ = yield self.execute(['-c', code.format(url)]) + code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" + errcode, out, _ = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) @defer.inlineCallbacks def test_scrapy_import(self): url = self.url('/text') - code = "fetch(scrapy.Request('{0}'))" - errcode, out, _ = yield self.execute(['-c', code.format(url)]) + code = f"fetch(scrapy.Request('{url}'))" + errcode, out, _ = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) @defer.inlineCallbacks diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 99c01c2b7..00d998388 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -16,7 +16,7 @@ class VersionTest(ProcessTest, unittest.TestCase): _, out, _ = yield self.execute([]) self.assertEqual( out.strip().decode(encoding), - "Scrapy %s" % scrapy.__version__, + f"Scrapy {scrapy.__version__}", ) @defer.inlineCallbacks diff --git a/tests/test_commands.py b/tests/test_commands.py index ee8a92604..5faaca738 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -42,7 +42,7 @@ class CommandSettings(unittest.TestCase): def test_settings_json_string(self): feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' - opts, args = self.parser.parse_args(args=['-s', 'FEEDS={}'.format(feeds_json), 'spider.py']) + opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) self.command.process_options(args, opts) self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) @@ -163,10 +163,10 @@ class StartprojectTemplatesTest(ProjectTest): pass assert exists(join(self.tmpl_proj, 'root_template')) - args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] + args = ['--set', f'TEMPLATES_DIR={self.tmpl}'] p, out, err = self.proc('startproject', self.project_name, *args) - self.assertIn("New Scrapy project '%s', using template directory" - % self.project_name, out) + self.assertIn(f"New Scrapy project '{self.project_name}', " + "using template directory", out) self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) @@ -247,7 +247,7 @@ class StartprojectTemplatesTest(ProjectTest): 'startproject', project_name, '--set', - 'TEMPLATES_DIR={}'.format(read_only_templates_dir), + f'TEMPLATES_DIR={read_only_templates_dir}', ), cwd=destination, env=self.env, @@ -320,7 +320,7 @@ class CommandTest(ProjectTest): super().setUp() self.call('startproject', self.project_name) self.cwd = join(self.temp_path, self.project_name) - self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name + self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings' class GenspiderCommandTest(CommandTest): @@ -334,14 +334,14 @@ class GenspiderCommandTest(CommandTest): assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) def test_template(self, tplname='crawl'): - args = ['--template=%s' % tplname] if tplname else [] + args = [f'--template={tplname}'] if tplname else [] spname = 'test_spider' p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) + self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module", out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn("Spider %r already exists in module" % spname, out) + self.assertIn(f"Spider {spname!r} already exists in module", out) modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) self.assertEqual(modify_time_after, modify_time_before) @@ -363,11 +363,11 @@ class GenspiderCommandTest(CommandTest): def test_same_name_as_project(self): self.assertEqual(2, self.call('genspider', self.project_name)) - assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name)) + assert not exists(join(self.proj_mod_path, 'spiders', f'{self.project_name}.py')) def test_same_filename_as_existing_spider(self, force=False): file_name = 'example' - file_path = join(self.proj_mod_path, 'spiders', '%s.py' % file_name) + file_path = join(self.proj_mod_path, 'spiders', f'{file_name}.py') self.assertEqual(0, self.call('genspider', file_name, 'example.com')) assert exists(file_path) @@ -383,14 +383,14 @@ class GenspiderCommandTest(CommandTest): if force: p, out, err = self.proc('genspider', '--force', file_name, 'example.com') - self.assertIn("Created spider %r using template \'basic\' in module" % file_name, out) + self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out) modify_time_after = getmtime(file_path) self.assertNotEqual(modify_time_after, modify_time_before) file_contents_after = open(file_path, 'r').read() self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn("%s already exists" % (file_path), out) + self.assertIn(f"{file_path} already exists", out) modify_time_after = getmtime(file_path) self.assertEqual(modify_time_after, modify_time_before) file_contents_after = open(file_path, 'r').read() @@ -410,7 +410,7 @@ class GenspiderStandaloneCommandTest(ProjectTest): file_name = 'example' file_path = join(self.temp_path, file_name + '.py') p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn("Created spider %r using template \'basic\' " % file_name, out) + self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out) assert exists(file_path) modify_time_before = getmtime(file_path) file_contents_before = open(file_path, 'r').read() @@ -418,14 +418,14 @@ class GenspiderStandaloneCommandTest(ProjectTest): if force: # use different template to ensure contents were changed p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com') - self.assertIn("Created spider %r using template \'crawl\' " % file_name, out) + self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out) modify_time_after = getmtime(file_path) self.assertNotEqual(modify_time_after, modify_time_before) file_contents_after = open(file_path, 'r').read() self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn("%s already exists" % join(self.temp_path, file_name + ".py"), out) + self.assertIn(f"{join(self.temp_path, file_name + '.py')} already exists", out) modify_time_after = getmtime(file_path) self.assertEqual(modify_time_after, modify_time_before) file_contents_after = open(file_path, 'r').read() diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 2e7e3ccc4..d0f4a68c2 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -393,7 +393,7 @@ class ContractsManagerTest(unittest.TestCase): return TestItem() with MockServer() as mockserver: - contract_doc = '@url {}'.format(mockserver.url('/status?n=200')) + contract_doc = f'@url {mockserver.url("/status?n=200")}' TestSameUrlSpider.parse_first.__doc__ = contract_doc TestSameUrlSpider.parse_second.__doc__ = contract_doc diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 642c24651..e703f45de 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -79,7 +79,7 @@ class CrawlTestCase(TestCase): total_time = times[-1] - times[0] average = total_time / (len(times) - 1) self.assertTrue(average > delay * tolerance, - "download delay too small: %s" % average) + f"download delay too small: {average}") # Ensure that the same test parameters would cause a failure if no # download delay is set. Otherwise, it means we are using a combination @@ -204,7 +204,7 @@ with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) + yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver) self.assertEqual(str(log).count("Got response 200"), 1) @defer.inlineCallbacks @@ -465,7 +465,7 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) for req_id in range(3): - self.assertIn("Got response 200, req_id %d" % req_id, str(log)) + self.assertIn(f"Got response 200, req_id {req_id}", str(log)) @defer.inlineCallbacks def test_response_ssl_certificate_none(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 7059f0892..0d3c42797 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -121,7 +121,7 @@ class FileTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_non_existent(self): - request = Request('file://%s' % self.mktemp()) + request = Request(f'file://{self.mktemp()}') d = self.download_request(request, Spider('foo')) return self.assertFailure(d, IOError) @@ -249,7 +249,7 @@ class HttpTestCase(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path) + return f"{self.scheme}://{self.host}:{self.portno}/{path}" def test_download(self): request = Request(self.getURL('file')) @@ -300,7 +300,7 @@ class HttpTestCase(unittest.TestCase): def test_host_header_not_in_request_headers(self): def _test(response): self.assertEqual( - response.body, to_bytes('%s:%d' % (self.host, self.portno))) + response.body, to_bytes(f'{self.host}:{self.portno}')) self.assertEqual(request.headers, {}) request = Request(self.getURL('host')) @@ -583,7 +583,7 @@ class Https11CustomCiphers(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path) + return f"{self.scheme}://{self.host}:{self.portno}/{path}" def test_download(self): request = Request(self.getURL('file')) @@ -678,7 +678,7 @@ class HttpProxyTestCase(unittest.TestCase): yield self.download_handler.close() def getURL(self, path): - return "http://127.0.0.1:%d/%s" % (self.portno, path) + return f"http://127.0.0.1:{self.portno}/{path}" def test_download_with_proxy(self): def _test(response): @@ -696,7 +696,7 @@ class HttpProxyTestCase(unittest.TestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.body, b'https://example.com') - http_proxy = '%s?noconnect' % self.getURL('') + http_proxy = f'{self.getURL("")}?noconnect' request = Request('https://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex(ScrapyDeprecationWarning, r'Using HTTPS proxies in the noconnect mode is deprecated'): @@ -977,7 +977,7 @@ class BaseFTPTestCase(unittest.TestCase): return deferred def test_ftp_download_success(self): - request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=self.req_meta) d = self.download_handler.download_request(request, None) @@ -989,7 +989,7 @@ class BaseFTPTestCase(unittest.TestCase): def test_ftp_download_path_with_spaces(self): request = Request( - url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum, + url=f"ftp://127.0.0.1:{self.portNum}/file with spaces.txt", meta=self.req_meta ) d = self.download_handler.download_request(request, None) @@ -1001,7 +1001,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): - request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt", meta=self.req_meta) d = self.download_handler.download_request(request, None) @@ -1015,7 +1015,7 @@ class BaseFTPTestCase(unittest.TestCase): os.close(f) meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) - request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) d = self.download_handler.download_request(request, None) @@ -1037,7 +1037,7 @@ class FTPTestCase(BaseFTPTestCase): meta = dict(self.req_meta) meta.update({"ftp_password": 'invalid'}) - request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) d = self.download_handler.download_request(request, None) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index a9190c62b..79f24c8a1 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -84,7 +84,7 @@ class DefaultsTest(ManagerTestCase): }) ret = self._download(request=req, response=resp) self.assertTrue(isinstance(ret, Request), - "Not redirected: {0!r}".format(ret)) + f"Not redirected: {ret!r}") self.assertEqual(to_bytes(ret.url), resp.headers['Location'], "Not redirected to location header") diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index dbae4d3ae..b2b5ce77d 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -28,7 +28,7 @@ class DecompressionMiddlewareTest(TestCase): for fmt in self.test_formats: rsp = self.test_responses[fmt] new = self.mw.process_response(None, rsp, self.spider) - error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__) + error_msg = f'Failed {fmt}, response type {type(new).__name__}' assert isinstance(new, XmlResponse), error_msg assert_samelines(self, new.body, self.uncompressed_body, fmt) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 299fb0eb8..0c6dcf2aa 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -324,7 +324,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware() as mw: for idx, (shouldcache, status, headers) in enumerate(responses): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) res1 = self._process_requestresponse(mw, req0, res0) res304 = res0.replace(status=304) @@ -343,7 +343,7 @@ class RFC2616PolicyTest(DefaultStorageTest): with self._middleware(HTTPCACHE_ALWAYS_STORE=True) as mw: for idx, (_, status, headers) in enumerate(responses): shouldcache = 'no-store' not in headers.get('Cache-Control', '') and status != 304 - req0 = Request('http://example2-%d.com' % idx) + req0 = Request(f'http://example2-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) res1 = self._process_requestresponse(mw, req0, res0) res304 = res0.replace(status=304) @@ -386,7 +386,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware() as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) # cache fresh response res1 = self._process_requestresponse(mw, req0, res0) @@ -423,7 +423,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware() as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0a = Response(req0.url, status=status, headers=headers) # cache expired response res1 = self._process_requestresponse(mw, req0, res0a) @@ -490,7 +490,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware(HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS=['no-cache', 'no-store']) as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) # cache fresh response res1 = self._process_requestresponse(mw, req0, res0) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 131332131..816ac1440 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -22,7 +22,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_redirect_3xx_permanent(self): def _test(method, status=301): - url = 'http://www.example.com/{}'.format(status) + url = f'http://www.example.com/{status}' url2 = 'http://www.example.com/redirected' req = Request(url, method=method) rsp = Response(url, headers={'Location': url2}, status=status) @@ -79,7 +79,7 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.method, 'GET') assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" - assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" # response without Location header but with status code is 3XX should be ignored del rsp.headers['Location'] @@ -207,8 +207,8 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.mw = MetaRefreshMiddleware.from_crawler(crawler) def _body(self, interval=5, url='http://example.org/newpage'): - html = """""" - return html.format(interval, url).encode('utf-8') + html = f"""""" + return html.encode('utf-8') def test_priority_adjust(self): req = Request('http://a.com') @@ -243,7 +243,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.assertEqual(req2.method, 'GET') assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" - assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" def test_max_redirect_times(self): self.mw.max_redirect_times = 1 diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 29357ba94..364ce0c89 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -94,7 +94,7 @@ class RetryTest(unittest.TestCase): ] for exc in exceptions: - req = Request('http://www.scrapytest.org/%s' % exc.__name__) + req = Request(f'http://www.scrapytest.org/{exc.__name__}') self._test_retry_exception(req, exc('foo')) stats = self.crawler.stats diff --git a/tests/test_engine.py b/tests/test_engine.py index 1b848ac72..3629aa1aa 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -127,8 +127,8 @@ def start_test_site(debug=False): port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: - print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish." - % port.getHost().port) + print(f"Test server running at http://localhost:{port.getHost().port}/ " + "- hit Ctrl-C to finish.") return port @@ -185,7 +185,7 @@ class CrawlerRun: self.deferred.callback(None) def geturl(self, path): - return "http://localhost:%s%s" % (self.portno, path) + return f"http://localhost:{self.portno}{path}" def getpath(self, url): u = urlparse(url) @@ -265,7 +265,7 @@ class EngineTest(unittest.TestCase): "/item1.html", "/item2.html", "/item999.html"] urls_visited = {rp[0].url for rp in self.run.respplug} urls_expected = {self.run.geturl(p) for p in must_be_visited} - assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited) + assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" def _assert_scheduled_requests(self, urls_to_visit=None): self.assertEqual(urls_to_visit, len(self.run.reqplug)) @@ -413,16 +413,19 @@ class StopDownloadEngineTest(EngineTest): yield self.run.run() log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - "Download stopped for from signal handler" - " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + f"Download stopped for " + "from signal handler" + " StopDownloadCrawlerRun.bytes_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - "Download stopped for from signal handler" - " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + f"Download stopped for " + "from signal handler" + " StopDownloadCrawlerRun.bytes_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - "Download stopped for from signal handler" - " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + f"Download stopped for " + "from signal handler" + " StopDownloadCrawlerRun.bytes_received")) self._assert_visited_urls() self._assert_scheduled_requests(urls_to_visit=9) self._assert_downloaded_responses() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 850485b5e..94568581a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -184,8 +184,7 @@ class FTPFeedStorageTest(unittest.TestCase): def test_uri_auth_quote(self): # RFC3986: 3.2.1. User Information pw_quoted = quote(string.punctuation, safe='') - st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted, - {}) + st = FTPFeedStorage(f'ftp://foo:{pw_quoted}@example.com/some_path', {}) self.assertEqual(st.password, string.punctuation) @@ -1230,7 +1229,7 @@ class FeedExportTest(FeedExportTestBase): print(log) for fmt in ['json', 'xml', 'csv']: - self.assertIn('Stored %s feed (2 items)' % fmt, str(log)) + self.assertIn(f'Stored {fmt} feed (2 items)', str(log)) @defer.inlineCallbacks def test_multiple_feeds_failing_logs_blocking_feed_storage(self): @@ -1251,7 +1250,7 @@ class FeedExportTest(FeedExportTestBase): print(log) for fmt in ['json', 'xml', 'csv']: - self.assertIn('Error storing %s feed (2 items)' % fmt, str(log)) + self.assertIn(f'Error storing {fmt} feed (2 items)', str(log)) class BatchDeliveriesTest(FeedExportTestBase): @@ -1582,10 +1581,8 @@ class BatchDeliveriesTest(FeedExportTestBase): chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) - prefix = 'tmp/{filename}'.format(filename=filename) - s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format( - bucket_name=s3_test_bucket_name, prefix=prefix - ) + prefix = f'tmp/{filename}' + s3_test_file_uri = f's3://{s3_test_bucket_name}/{prefix}/%(batch_time)s.json' storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) settings = Settings({ 'FEEDS': { diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 624dd9ab8..41afa2896 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -657,7 +657,7 @@ class SelectJmesTestCase(unittest.TestCase): self.assertEqual( test, expected, - msg='test "{}" got {} expected {}'.format(tl, test, expected) + msg=f'test "{tl}" got {test} expected {expected}' ) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 41ff3651d..dc5be398f 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -20,7 +20,7 @@ class CustomItem(Item): name = Field() def __str__(self): - return "name: %s" % self['name'] + return f"name: {self['name']}" class LogFormatterTestCase(unittest.TestCase): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b2b75ef20..e3e46db07 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -50,7 +50,7 @@ class TestMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] + return [f'tests.test_middleware.{x}' for x in ['M1', 'MOff', 'M3']] def _add_middleware(self, mw): super()._add_middleware(mw) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 9af5affec..55fcfa7ba 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -123,10 +123,10 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4) self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4) self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1) - self.assertEqual(crawler.stats.get_value('downloader/response_status_count/%d' % code), 3) + self.assertEqual(crawler.stats.get_value(f'downloader/response_status_count/{code}'), 3) # check that logs do show the failure on the file downloads - file_dl_failure = 'File (code: %d): Error downloading file from' % code + file_dl_failure = f'File (code: {code}): Error downloading file from' self.assertEqual(logs.count(file_dl_failure), 3) # check that no files were written to the media store diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index b19b4ff2a..1dd7031fe 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -167,7 +167,7 @@ class FilesPipelineTestCase(unittest.TestCase): """ class CustomFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, item=None): - return 'full/%s' % item.get('path') + return f'full/{item.get("path")}' file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path item = dict(path='path-to-store-file') @@ -495,7 +495,7 @@ class TestFTPFileStore(unittest.TestCase): self.assertIn('last_modified', stat) self.assertIn('checksum', stat) self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6') - path = '%s/%s' % (store.basedir, path) + path = f'{store.basedir}/{path}' content = get_ftp_content_and_delete( path, store.host, store.port, store.username, store.password, store.USE_ACTIVE_MODE) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 082e9ee21..ad138a2dc 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -128,11 +128,11 @@ class DeprecatedImagesPipeline(ImagesPipeline): def image_key(self, url): image_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return 'empty/%s.jpg' % (image_guid) + return f'empty/{image_guid}.jpg' def thumb_key(self, url, thumb_id): thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid) + return f'thumbsup/{thumb_id}/{thumb_guid}.jpg' class ImagesPipelineTestCaseFieldsMixin: diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index a56e3c39a..d3f58634e 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -37,14 +37,14 @@ sys.exit(mitmdump()) '-c', script, '--listen-host', '127.0.0.1', '--listen-port', '0', - '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass), + '--proxyauth', f'{self.auth_user}:{self.auth_pass}', '--certs', cert_path, '--ssl-insecure', ], stdout=PIPE, env=get_testenv()) line = self.proc.stdout.readline().decode('utf-8') host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1) - address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port) + address = f'http://{self.auth_user}:{self.auth_pass}@{host_port}' return address def stop(self): @@ -118,7 +118,7 @@ class ProxyConnectTestCase(TestCase): def _assert_got_response_code(self, code, log): print(log) - self.assertEqual(str(log).count('Crawled (%d)' % code), 1) + self.assertEqual(str(log).count(f'Crawled ({code})'), 1) def _assert_got_tunnel_error(self, log): print(log) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index b60b7c579..907117468 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -79,7 +79,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_response_error(self): for status in ("404", "500"): - url = self.mockserver.url("/status?n={}".format(status)) + url = self.mockserver.url(f"/status?n={status}") crawler = CrawlerRunner().create_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] @@ -135,7 +135,7 @@ class CrawlTestCase(TestCase): self.assertEqual(signal_params["request"].url, OVERRIDEN_URL) log.check_present( - ("scrapy.core.engine", "DEBUG", "Crawled (200) (referer: None)".format(OVERRIDEN_URL)), + ("scrapy.core.engine", "DEBUG", f"Crawled (200) (referer: None)"), ) @defer.inlineCallbacks diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index a175f88ca..c07d3a99c 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -17,7 +17,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_filename(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_content_disposition(self): mappings = [ @@ -32,7 +32,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_content_disposition(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_content_type(self): mappings = [ @@ -47,7 +47,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_content_type(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_body(self): mappings = [ @@ -58,7 +58,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_body(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_headers(self): mappings = [ @@ -70,7 +70,7 @@ class ResponseTypesTest(unittest.TestCase): for source, cls in mappings: source = Headers(source) retcls = responsetypes.from_headers(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_args(self): # TODO: add more tests that check precedence between the different arguments @@ -86,7 +86,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_args(**source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_custom_mime_types_loaded(self): # check that mime.types files shipped with scrapy are loaded diff --git a/tests/test_selector.py b/tests/test_selector.py index 62036ad8c..cff8d0393 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -88,7 +88,7 @@ class SelectorTestCase(unittest.TestCase): """Check that classes are using slots and are weak-referenceable""" x = Selector(text='') weakref.ref(x) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % x.__class__.__name__ + assert not hasattr(x, '__dict__'), f"{x.__class__.__name__} does not use __slots__" def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, 'received both response and text'): diff --git a/tests/test_signals.py b/tests/test_signals.py index d6ae526be..a43f00b27 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -13,7 +13,7 @@ class ItemSpider(Spider): def start_requests(self): for index in range(10): - yield Request(self.mockserver.url('/status?n=200&id=%d' % index), + yield Request(self.mockserver.url(f'/status?n=200&id={index}'), meta={'index': index}) def parse(self, response): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 79eda35b3..2f454addc 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -163,11 +163,11 @@ class GeneratorOutputChainSpider(Spider): class _GeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') yield r def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return None @@ -175,12 +175,12 @@ class _GeneratorDoNothingMiddleware: class GeneratorFailMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') yield r raise LookupError() def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} @@ -192,11 +192,11 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware): class GeneratorRecoverMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') yield r def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} @@ -229,12 +229,12 @@ class _NotGeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') out.append(r) return out def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return None @@ -243,13 +243,13 @@ class NotGeneratorFailMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') out.append(r) raise ReferenceError() return out def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] @@ -262,12 +262,12 @@ class NotGeneratorRecoverMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') out.append(r) return out def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 6b05c8771..f5d684d3f 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -16,7 +16,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): try: Request(**result) except TypeError as e: - self.fail("Request kwargs are not correct {}".format(e)) + self.fail(f"Request kwargs are not correct {e}") def test_get(self): curl_command = "curl http://example.org/" diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index aa18ef1f3..e4bccf30e 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -299,7 +299,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): cache = LocalWeakReferencedCache() refs = [] for x in range(max): - refs.append(Request('https://example.org/{}'.format(x))) + refs.append(Request(f'https://example.org/{x}')) cache[refs[-1]] = x self.assertEqual(len(cache), max) for i, r in enumerate(refs): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 8c84331b9..e60242a3b 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -40,15 +40,15 @@ class MustbeDeferredTest(unittest.TestCase): def cb1(value, arg1, arg2): - return "(cb1 %s %s %s)" % (value, arg1, arg2) + return f"(cb1 {value} {arg1} {arg2})" def cb2(value, arg1, arg2): - return defer.succeed("(cb2 %s %s %s)" % (value, arg1, arg2)) + return defer.succeed(f"(cb2 {value} {arg1} {arg2})") def cb3(value, arg1, arg2): - return "(cb3 %s %s %s)" % (value, arg1, arg2) + return f"(cb3 {value} {arg1} {arg2})" def cb_fail(value, arg1, arg2): @@ -56,7 +56,7 @@ def cb_fail(value, arg1, arg2): def eb1(failure, arg1, arg2): - return "(eb1 %s %s %s)" % (failure.value.__class__.__name__, arg1, arg2) + return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})" class DeferUtilsTest(unittest.TestCase): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 50190d4d1..79f5a2bbe 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -409,7 +409,7 @@ class TestHelper(unittest.TestCase): def _assert_type_and_value(self, a, b, obj): self.assertTrue(type(a) is type(b), - 'Got {}, expected {} for {!r}'.format(type(a), type(b), obj)) + f'Got {type(a)}, expected {type(b)} for { obj!r}') self.assertEqual(a, b) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 2f885a0e8..144c7bd76 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -213,7 +213,7 @@ def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) assert url.startswith(args[1]), \ - 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (args[0], url, args[1]) + f'Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`' return do_expected @@ -254,7 +254,7 @@ for k, args in enumerate( start=1, ): t_method = create_guess_scheme_t(args) - t_method.__name__ = 'test_uri_%03d' % k + t_method.__name__ = f'test_uri_{k:03}' setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation @@ -269,7 +269,7 @@ for k, args in enumerate( start=1, ): t_method = create_skipped_scheme_t(args) - t_method.__name__ = 'test_uri_skipped_%03d' % k + t_method.__name__ = f'test_uri_skipped_{k:03}' setattr(GuessSchemeTest, t_method.__name__, t_method) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index ee64d455c..a60181a3a 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -253,7 +253,7 @@ class WebClientTestCase(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return "http://127.0.0.1:%d/%s" % (self.portno, path) + return f"http://127.0.0.1:{self.portno}/{path}" def testPayload(self): s = "0123456789" * 10 @@ -265,7 +265,7 @@ class WebClientTestCase(unittest.TestCase): # it should extract from url return defer.gatherResults([ getPage(self.getURL("host")).addCallback( - self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)), + self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")), getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback( self.assertEqual, to_bytes("www.example.com"))]) @@ -298,7 +298,7 @@ class WebClientTestCase(unittest.TestCase): """ d = getPage(self.getURL("host"), timeout=100) d.addCallback( - self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)) + self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")) return d def test_timeoutTriggering(self): @@ -376,7 +376,7 @@ class WebClientSSLTestCase(unittest.TestCase): interface="127.0.0.1") def getURL(self, path): - return "https://127.0.0.1:%d/%s" % (self.portno, path) + return f"https://127.0.0.1:{self.portno}/{path}" def setUp(self): self.tmpname = self.mktemp() From 560c335c0782ec9a834a83abfa9dfbaa8fd67fed Mon Sep 17 00:00:00 2001 From: maranqz Date: Wed, 26 Aug 2020 14:00:51 +0300 Subject: [PATCH 269/568] Add errors parameter in documentation. --- docs/topics/exporters.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 11ef5b2a6..3f8906326 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -292,7 +292,7 @@ XmlItemExporter CsvItemExporter --------------- -.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs) +.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs) Exports items in CSV format to the given file-like object. If the :attr:`fields_to_export` attribute is set, it will be used to define the @@ -311,6 +311,10 @@ CsvItemExporter multi-valued fields, if found. :type include_headers_line: str + :param errors: The optional string that specifies how encoding and decoding errors are to be handled. + For more information see `documentation `_. + :type errors: str + The additional keyword arguments of this ``__init__`` method are passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the :func:`csv.writer` function, so you can use any :func:`csv.writer` function From cf50561b8696b645940ae33ba9b168ef97580477 Mon Sep 17 00:00:00 2001 From: nyov Date: Wed, 26 Aug 2020 11:08:14 +0000 Subject: [PATCH 270/568] Allow passing classes directly in Settings (#3873) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/settings.rst | 26 +++++++++++++++++++++ scrapy/utils/deprecate.py | 2 +- scrapy/utils/misc.py | 14 +++++++++-- tests/test_crawler.py | 2 +- tests/test_downloader_handlers.py | 6 ++--- tests/test_feedexport.py | 22 +++++++++--------- tests/test_middleware.py | 2 +- tests/test_settings/__init__.py | 32 ++++++++++++++++++++++++++ tests/test_spidermiddleware_referer.py | 2 +- tests/test_utils_misc/__init__.py | 15 ++++++++++-- 10 files changed, 101 insertions(+), 22 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 618b9989e..2924c0566 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -98,6 +98,32 @@ class. The global defaults are located in the ``scrapy.settings.default_settings`` module and documented in the :ref:`topics-settings-ref` section. + +Import paths and classes +======================== + +.. versionadded:: VERSION + +When a setting references a callable object to be imported by Scrapy, such as a +class or a function, there are two different ways you can specify that object: + +- As a string containing the import path of that object + +- As the object itself + +For example:: + + from mybot.pipelines.validate import ValidateMyItem + ITEM_PIPELINES = { + # passing the classname... + ValidateMyItem: 300, + # ...equals passing the class path + 'mybot.pipelines.validate.ValidateMyItem': 300, + } + +.. note:: Passing non-callable objects is not supported. + + How to access settings ====================== diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3c8e3c8b5..8277a3c8f 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -135,7 +135,7 @@ DEPRECATION_RULES = [ def update_classpath(path): """Update a deprecated path from an object with its new location""" for prefix, replacement in DEPRECATION_RULES: - if path.startswith(prefix): + if isinstance(path, str) and path.startswith(prefix): new_path = path.replace(prefix, replacement, 1) warnings.warn("`{}` class is deprecated, use `{}` instead".format(path, new_path), ScrapyDeprecationWarning) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index bd400bd30..d5d1f301c 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -39,10 +39,20 @@ def arg_to_iter(arg): def load_object(path): """Load an object given its absolute object path, and return it. - object can be the import path of a class, function, variable or an - instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware' + The object can be the import path of a class, function, variable or an + instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'. + + If ``path`` is not a string, but is a callable object, such as a class or + a function, then return it as is. """ + if not isinstance(path, str): + if callable(path): + return path + else: + raise TypeError("Unexpected argument type, expected string " + "or object, got: %s" % type(path)) + try: dot = path.rindex('.') except ValueError: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 7c2e251a9..85035a220 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -142,7 +142,7 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): def test_spider_manager_verify_interface(self): settings = Settings({ - 'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface' + 'SPIDER_LOADER_CLASS': SpiderLoaderWithWrongInterface, }) with warnings.catch_warnings(record=True) as w: self.assertRaises(AttributeError, CrawlerRunner, settings) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 7059f0892..e50bdb391 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -61,7 +61,7 @@ class OffDH: class LoadTestCase(unittest.TestCase): def test_enabled_handler(self): - handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'} + handlers = {'scheme': DummyDH} crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) @@ -69,7 +69,7 @@ class LoadTestCase(unittest.TestCase): self.assertNotIn('scheme', dh._notconfigured) def test_not_configured_handler(self): - handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'} + handlers = {'scheme': OffDH} crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) @@ -87,7 +87,7 @@ class LoadTestCase(unittest.TestCase): self.assertIn('scheme', dh._notconfigured) def test_lazy_handlers(self): - handlers = {'scheme': 'tests.test_downloader_handlers.DummyLazyDH'} + handlers = {'scheme': DummyLazyDH} crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 850485b5e..689d25fef 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -845,7 +845,7 @@ class FeedExportTest(FeedExportTestBase): self._random_temp_filename(): {'format': 'xml'}, self._random_temp_filename(): {'format': 'csv'}, }, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'}, + 'FEED_STORAGES': {'file': LogOnStoreFileStorage}, 'FEED_STORE_EMPTY': False } @@ -1189,8 +1189,8 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_init_exporters_storages_with_crawler(self): settings = { - 'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.FromCrawlerCsvItemExporter'}, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.FromCrawlerFileFeedStorage'}, + 'FEED_EXPORTERS': {'csv': FromCrawlerCsvItemExporter}, + 'FEED_STORAGES': {'file': FromCrawlerFileFeedStorage}, 'FEEDS': { self._random_temp_filename(): {'format': 'csv'}, }, @@ -1219,7 +1219,7 @@ class FeedExportTest(FeedExportTestBase): self._random_temp_filename(): {'format': 'xml'}, self._random_temp_filename(): {'format': 'csv'}, }, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.DummyBlockingFeedStorage'}, + 'FEED_STORAGES': {'file': DummyBlockingFeedStorage}, } items = [ {'foo': 'bar1', 'baz': ''}, @@ -1240,7 +1240,7 @@ class FeedExportTest(FeedExportTestBase): self._random_temp_filename(): {'format': 'xml'}, self._random_temp_filename(): {'format': 'csv'}, }, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.FailingBlockingFeedStorage'}, + 'FEED_STORAGES': {'file': FailingBlockingFeedStorage}, } items = [ {'foo': 'bar1', 'baz': ''}, @@ -1667,7 +1667,7 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.StdoutFeedStorageWithoutFeedOptions' + 'file': StdoutFeedStorageWithoutFeedOptions }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1708,7 +1708,7 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.FileFeedStorageWithoutFeedOptions' + 'file': FileFeedStorageWithoutFeedOptions }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1756,7 +1756,7 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptions' + 'file': S3FeedStorageWithoutFeedOptions }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1784,7 +1784,7 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptionsWithFromCrawler' + 'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1833,7 +1833,7 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptions' + 'file': FTPFeedStorageWithoutFeedOptions }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1861,7 +1861,7 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptionsWithFromCrawler' + 'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler }, } crawler = get_crawler(settings_dict=settings_dict) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b2b75ef20..8651431b5 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -50,7 +50,7 @@ class TestMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] + return [M1, MOff, M3] def _add_middleware(self, mw): super()._add_middleware(mw) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 6e56a28f5..916fe012a 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -385,6 +385,38 @@ class SettingsTest(unittest.TestCase): self.assertIn('key', mydict) self.assertEqual(mydict['key'], 'val') + def test_passing_objects_as_values(self): + from scrapy.core.downloader.handlers.file import FileDownloadHandler + from scrapy.utils.misc import create_instance + from scrapy.utils.test import get_crawler + + class TestPipeline(): + def process_item(self, i, s): + return i + + settings = Settings({ + 'ITEM_PIPELINES': { + TestPipeline: 800, + }, + 'DOWNLOAD_HANDLERS': { + 'ftp': FileDownloadHandler, + }, + }) + + self.assertIn('ITEM_PIPELINES', settings.attributes) + + mypipeline, priority = settings.getdict('ITEM_PIPELINES').popitem() + self.assertEqual(priority, 800) + self.assertEqual(mypipeline, TestPipeline) + self.assertIsInstance(mypipeline(), TestPipeline) + self.assertEqual(mypipeline().process_item('item', None), 'item') + + myhandler = settings.getdict('DOWNLOAD_HANDLERS').pop('ftp') + self.assertEqual(myhandler, FileDownloadHandler) + myhandler_instance = create_instance(myhandler, None, get_crawler()) + self.assertIsInstance(myhandler_instance, FileDownloadHandler) + self.assertTrue(hasattr(myhandler_instance, 'download_request')) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 5141f47af..9456b01d4 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -385,7 +385,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): class TestSettingsCustomPolicy(TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + settings = {'REFERRER_POLICY': CustomPythonOrgPolicy} scenarii = [ ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 9bb996d27..e95a3a316 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -12,11 +12,22 @@ __doctests__ = ['scrapy.utils.misc'] class UtilsMiscTestCase(unittest.TestCase): - def test_load_object(self): + def test_load_object_class(self): + obj = load_object(Field) + self.assertIs(obj, Field) + obj = load_object('scrapy.item.Field') + self.assertIs(obj, Field) + + def test_load_object_function(self): + obj = load_object(load_object) + self.assertIs(obj, load_object) obj = load_object('scrapy.utils.misc.load_object') - assert obj is load_object + self.assertIs(obj, load_object) + + def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') + self.assertRaises(TypeError, load_object, dict()) def test_walk_modules(self): mods = walk_modules('tests.test_utils_misc.test_walk_modules') From 29725e4b58bfe417388b5ce2c2b4f1c587a465da Mon Sep 17 00:00:00 2001 From: maranqz Date: Wed, 26 Aug 2020 14:38:37 +0300 Subject: [PATCH 271/568] Fix tabulation of rst and change documentation link. --- docs/topics/exporters.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 3f8906326..0203def74 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -311,9 +311,10 @@ CsvItemExporter multi-valued fields, if found. :type include_headers_line: str - :param errors: The optional string that specifies how encoding and decoding errors are to be handled. - For more information see `documentation `_. - :type errors: str + :param errors: The optional string that specifies how encoding and decoding + errors are to be handled. For more information see + :class:`io.TextIOWrapper`. + :type errors: str The additional keyword arguments of this ``__init__`` method are passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the From c77450990d9f78e68380d724b4a8c15e7b99c995 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 11:41:24 +0000 Subject: [PATCH 272/568] Update scrapy/commands/version.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/commands/version.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index dc8087043..1237610cb 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -23,8 +23,7 @@ class Command(ScrapyCommand): if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) - patt = f"%-{width}s : %s" for name, version in versions: - print(patt % (name, version)) + print(f"{name:<{width}} : {version}") else: print(f"Scrapy {scrapy.__version__}") From 92dfa7176dbadcb92e4aae9d3f2c2b02c6e52a4a Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 11:43:40 +0000 Subject: [PATCH 273/568] Update scrapy/extensions/statsmailer.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/extensions/statsmailer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 997e74fc9..bcdbaff24 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -28,7 +28,7 @@ class StatsMailer: def spider_closed(self, spider): spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" - body += "\n".join(f"{i:<50} : {self.stats.get_stats()[i]}" for i in self.stats.get_stats()) + body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) body += f"\n\n{spider.name} stats\n\n" - body += "\n".join(f"{i:<50} : {spider_stats[i]}" for i in spider_stats) + body += "\n".join(f"{k:<50} : {v}" for k, v in spider_stats.items()) return self.mail.send(self.recipients, f"Scrapy stats for: {spider.name}", body) From ea03e4254fbaff71ce584a461344eb0eb9aa7e09 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 11:43:52 +0000 Subject: [PATCH 274/568] Update scrapy/http/request/form.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index c90d68fa1..2815303a2 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -204,7 +204,7 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = './/*' + ''.join(f'[@{key}="{clickdata[key]}"]' for key in clickdata) + xpath = './/*' + ''.join(f'[@{k}="{v}"]' for k, v in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') From 9aaddcde0af5910e33fbba253777149e04f021f8 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 11:44:20 +0000 Subject: [PATCH 275/568] Update scrapy/utils/conf.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 05cd5f25c..afd8f5374 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -42,8 +42,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): raise ValueError(f'Invalid value {value} for component {name}, ' - 'please provide a real number or None instead' - ) + 'please provide a real number or None instead') # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): From 2ca8dfb4b08ad416716dc18d03983cc159cc5634 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 13:49:39 +0200 Subject: [PATCH 276/568] revert f-string changes for files under sep/ Issue #4324 --- sep/sep-002.rst | 2 +- sep/sep-004.rst | 4 ++-- sep/sep-014.rst | 29 +++++++++++++++-------------- sep/sep-018.rst | 22 +++++++++++----------- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/sep/sep-002.rst b/sep/sep-002.rst index 2e8a28340..c467cb402 100644 --- a/sep/sep-002.rst +++ b/sep/sep-002.rst @@ -30,7 +30,7 @@ Proposed Implementation if hasattr(value, '__iter__'): # str/unicode not allowed return [self._field.to_python(v) for v in value] else: - raise TypeError(f"Expected iterable, got {type(value).__name__}") + raise TypeError("Expected iterable, got %s" % type(value).__name__) def get_default(self): # must return a new copy to avoid unexpected behaviors with mutable defaults diff --git a/sep/sep-004.rst b/sep/sep-004.rst index b9f5e556f..05b0eb99c 100644 --- a/sep/sep-004.rst +++ b/sep/sep-004.rst @@ -11,7 +11,7 @@ SEP-004: Library API ==================== .. note:: the library API has been implemented, but slightly different from proposed in this SEP. You can run a Scrapy crawler inside a Twisted - reactor, but not outside it. + reactor, but not outside it. Introduction ============ @@ -49,7 +49,7 @@ Here's a simple proof-of-concept code of such script: cr = Crawler(start_urls, callback=parse_start_page) cr.run() # blocking call - this populates scraped_items - print(f"{len(scraped_items)} items scraped") + print "%d items scraped" % len(scraped_items) # ... do something more interesting with scraped_items ... The behaviour of the Scrapy crawler would be controller by the Scrapy settings, diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 4e3340521..8ca81824d 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -21,7 +21,7 @@ Current flaws and inconsistencies 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) 3. Isn't possible to crawl an url directly from command line because the Spider - does not know which callback use. + does not know which callback use. These flaws will be corrected by the changes proposed in this SEP. @@ -55,7 +55,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. This is an enhancement to ``LinkExtractors`` which returns urls (links), -Request Extractors return Request objects. +Request Extractors return Request objects. Request Processors ------------------ @@ -142,7 +142,7 @@ Custom Processor and External Callback # Callback defined out of spider def my_external_callback(response): - # process item + # process item pass class SampleSpider(CrawlSpider): @@ -233,7 +233,7 @@ Request/Response Matchers def matches_request(self, request): """Returns True if Request's url matches initial url""" - return self.matches_url(request.url) + return self.matches_url(request.url) def matches_response(self, response): """REturns True if Response's url matches initial url""" @@ -305,14 +305,14 @@ Request Extractor for req in self.requests: req.meta.setdefault('link_text', '') req.meta['link_text'] = str_to_unicode(req.meta['link_text'], - encoding) + encoding) def reset(self): """Reset state""" FixedSGMLParser.reset(self) self.requests = [] self.base_url = None - + def unknown_starttag(self, tag, attrs): """Process unknown start tag""" if 'base' tag: @@ -376,7 +376,7 @@ Request Processor #!python # - # Request Processors + # Request Processors # Processors receive list of requests and return list of requests # """Request Processors""" @@ -390,7 +390,7 @@ Request Processor # replace in-place req.url = canonicalize_url(req.url) yield req - + class Unique(object): """Filter duplicate Requests""" @@ -455,9 +455,9 @@ Request Processor """Initialize allow/deny attributes""" _re_type = type(re.compile('', 0)) - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] def __call__(self, requests): @@ -524,7 +524,7 @@ Rules Manager # # Handles rules matcher/callbacks # Resolve rule for given response - # + # class RulesManager(object): """Rules Manager""" def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): @@ -542,8 +542,8 @@ Rules Manager # instance default matcher matcher = default_matcher(rule.matcher) else: - raise ValueError('Not valid matcher given ' - f'{rule.matcher!r} in {rule!r}') + raise ValueError('Not valid matcher given %r in %r' \ + % (rule.matcher, rule)) # prepare callback if callable(rule.callback): @@ -553,7 +553,8 @@ Rules Manager callback = getattr(spider, rule.callback) if not callable(callback): - raise AttributeError(f'Invalid callback {callback!r} can not be resolved') + raise AttributeError('Invalid callback %r can not be resolved' \ + % callback) else: callback = None diff --git a/sep/sep-018.rst b/sep/sep-018.rst index d0169b81e..fe707923a 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -171,7 +171,7 @@ the same spider: #!python class MySpider(BaseSpider): - middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), + middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), ItemIdSetter(), OffsiteMiddleware()] allowed_domains = ['example.com', 'sub.example.com'] @@ -196,7 +196,7 @@ the same spider: # extract item from response return item -The Spider Middleware that implements spider code +The Spider Middleware that implements spider code ================================================= There's gonna be one middleware that will take care of calling the proper @@ -324,7 +324,7 @@ Another example could be for building URL canonicalizers: class CanonializeUrl(object): def process_request(self, request, response, spider): - curl = canonicalize_url(request.url, + curl = canonicalize_url(request.url, rules=spider.canonicalization_rules) return request.replace(url=curl) @@ -332,7 +332,7 @@ Another example could be for building URL canonicalizers: class MySpider(BaseSpider): middlewares = [CanonicalizeUrl()] - canonicalization_rules = ['sort-query-args', + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] # ... @@ -414,7 +414,7 @@ A spider middleware to avoid visiting pages forbidden by robots.txt: if netloc in info.pending: res = None else: - robotsurl = f"{url.scheme}://{netloc}/robots.txt" + robotsurl = "%s://%s/robots.txt" % (url.scheme, netloc) meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]} res = Request(robotsurl, callback=self.parse_robots, meta=meta, priority=self.REQUEST_PRIORITY) @@ -474,7 +474,7 @@ This is a port of the Offsite middleware to the new spider middleware API: if host and host not in info.hosts_seen: spider.log("Filtered offsite request to %r: %s" % (host, request)) info.hosts_seen.add(host) - + def should_follow(self, request, spider): info = self.spiders[spider] # hostname can be None for wrong urls (like javascript links) @@ -484,7 +484,7 @@ This is a port of the Offsite middleware to the new spider middleware API: def get_host_regex(self, spider): """Override this method to implement a different offsite policy""" domains = [d.replace('.', r'\.') for d in spider.allowed_domains] - regex = fr'^(.*\.)?({"|".join(domains)})$' + regex = r'^(.*\.)?(%s)$' % '|'.join(domains) return re.compile(regex) def spider_opened(self, spider): @@ -570,7 +570,7 @@ A middleware to filter out requests already seen: self.dupefilter = load_object(clspath)() dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - + def enqueue_request(self, spider, request): seen = self.dupefilter.request_seen(spider, request) if not seen or request.dont_filter: @@ -601,8 +601,8 @@ A middleware to Scrape data using Parsley as described in UsingParsley for name in parslet.keys(): self.fields[name] = Field() super(ParsleyItem, self).__init__(*a, **kw) - self.item_class = ParsleyItem - self.parsley = PyParsley(parslet, output='python') + self.item_class = ParsleyItem + self.parsley = PyParsley(parslet, output='python') def process_response(self, response, request, spider): return self.item_class(self.parsly.parse(string=response.body)) @@ -627,7 +627,7 @@ Resolved: not the original one (think of redirections), but it does carry the ``meta`` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be - the deserialized request from the persistent scheduler queue. + the deserialized request from the persistent scheduler queue. - No - this would make implementation more complex and we're not sure it's really needed From 450ba6b51f7cc3cd89a92e8ff4d9e105865eb862 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 26 Aug 2020 17:20:59 +0530 Subject: [PATCH 277/568] fix(typo): stream -> streams, use isinstance --- scrapy/core/http2/protocol.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index c6f00423a..6647ae0b7 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -90,7 +90,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # all requests in a pool and send them as the connection is made self._pending_request_stream_pool: deque = deque() - # Counter to keep track of opened stream. This counter + # Counter to keep track of opened streams. This counter # is used to make sure that not more than MAX_CONCURRENT_STREAMS # streams are opened which leads to ProtocolError # We use simple FIFO policy to handle pending requests @@ -218,7 +218,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" negotiated_protocol = self.transport.negotiatedProtocol - if type(negotiated_protocol) is bytes: + if isinstance(negotiated_protocol, bytes): negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') if negotiated_protocol != 'h2': # Here we have not initiated the connection yet From a8114d3731cfd1ef8fdb808b1563a7288b8a2e90 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 26 Aug 2020 09:00:36 -0300 Subject: [PATCH 278/568] Typing: annotate a few Spider attributes --- scrapy/spiders/__init__.py | 5 ++-- scrapy/spiders/crawl.py | 3 ++- setup.cfg | 51 -------------------------------------- tests/spiders.py | 2 +- 4 files changed, 6 insertions(+), 55 deletions(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 12b4fba09..a66d65846 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -5,6 +5,7 @@ See documentation in docs/topics/spiders.rst """ import logging import warnings +from typing import Optional from scrapy import signals from scrapy.http import Request @@ -18,8 +19,8 @@ class Spider(object_ref): class. """ - name = None - custom_settings = None + name: Optional[str] = None + custom_settings: Optional[dict] = None def __init__(self, name=None, **kwargs): if name is not None: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index c9fbce08d..bc4551a54 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -7,6 +7,7 @@ See documentation in docs/topics/spiders.rst import copy import warnings +from typing import Sequence from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse @@ -72,7 +73,7 @@ class Rule: class CrawlSpider(Spider): - rules = () + rules: Sequence[Rule] = () def __init__(self, *a, **kw): super().__init__(*a, **kw) diff --git a/setup.cfg b/setup.cfg index 3a624ec94..8101443e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,9 +16,6 @@ ignore_errors = True [mypy-scrapy.commands] ignore_errors = True -[mypy-scrapy.commands.bench] -ignore_errors = True - [mypy-scrapy.commands.parse] ignore_errors = True @@ -28,9 +25,6 @@ ignore_errors = True [mypy-scrapy.contracts] ignore_errors = True -[mypy-scrapy.core.spidermw] -ignore_errors = True - [mypy-scrapy.interfaces] ignore_errors = True @@ -70,15 +64,6 @@ ignore_errors = True [mypy-tests.mocks.dummydbm] ignore_errors = True -[mypy-tests.spiders] -ignore_errors = True - -[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception] -ignore_errors = True - -[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal] -ignore_errors = True - [mypy-tests.test_command_fetch] ignore_errors = True @@ -94,9 +79,6 @@ ignore_errors = True [mypy-tests.test_contracts] ignore_errors = True -[mypy-tests.test_crawler] -ignore_errors = True - [mypy-tests.test_downloader_handlers] ignore_errors = True @@ -127,53 +109,20 @@ ignore_errors = True [mypy-tests.test_pipeline_images] ignore_errors = True -[mypy-tests.test_pipelines] -ignore_errors = True - -[mypy-tests.test_request_attribute_binding] -ignore_errors = True - [mypy-tests.test_request_cb_kwargs] ignore_errors = True -[mypy-tests.test_request_left] -ignore_errors = True - [mypy-tests.test_scheduler] ignore_errors = True -[mypy-tests.test_signals] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.nested.spider4] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.spider1] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.spider2] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.spider3] -ignore_errors = True - [mypy-tests.test_spidermiddleware_httperror] ignore_errors = True -[mypy-tests.test_spidermiddleware_output_chain] -ignore_errors = True - [mypy-tests.test_spidermiddleware_referer] ignore_errors = True -[mypy-tests.test_utils_reqser] -ignore_errors = True - [mypy-tests.test_utils_serialize] ignore_errors = True -[mypy-tests.test_utils_spider] -ignore_errors = True - [mypy-tests.test_utils_url] ignore_errors = True diff --git a/tests/spiders.py b/tests/spiders.py index 63bd726fb..8a0ee44b7 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -255,7 +255,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): A CrawlSpider which overrides the 'parse' method """ name = 'crawl_spider_with_parse_method' - custom_settings = { + custom_settings: dict = { 'RETRY_HTTP_CODES': [], # no need to retry } rules = ( From 5ab1a318e8a8245cad694edc297d94ac80a90760 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 15:11:46 +0200 Subject: [PATCH 279/568] test: list appears in ValueError Exception message Issue #4324 --- scrapy/utils/conf.py | 2 +- tests/test_utils_conf.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index afd8f5374..4e7a9967e 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -17,7 +17,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): def _check_components(complist): if len({convert(c) for c in complist}) != len(complist): - raise ValueError('Some paths in {complist!r} convert to the same object, ' + raise ValueError(f'Some paths in {complist!r} convert to the same object, ' 'please update your settings') def _map_keys(compdict): diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index ccc65c4fd..061bc8c7c 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -50,8 +50,9 @@ class BuildComponentListTest(unittest.TestCase): def test_duplicate_components_in_list(self): duplicate_list = ['a', 'b', 'a'] - self.assertRaises(ValueError, build_component_list, None, - duplicate_list, convert=lambda x: x) + with self.assertRaises(ValueError) as cm: + build_component_list(None, duplicate_list, convert=lambda x: x) + self.assertIn(str(duplicate_list), str(cm.exception)) def test_duplicate_components_in_basesettings(self): # Higher priority takes precedence From dd378b4bb1499b9e8da7b193f19ae6a8d81b5a2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 27 Aug 2020 11:07:58 +0200 Subject: [PATCH 280/568] Generate localhost keys for tests on the fly --- .gitignore | 2 ++ conftest.py | 6 ++++ tests/keys/__init__.py | 63 ++++++++++++++++++++++++++++++++++++++++ tests/keys/localhost.crt | 20 ------------- tests/keys/localhost.key | 28 ------------------ 5 files changed, 71 insertions(+), 48 deletions(-) create mode 100644 tests/keys/__init__.py delete mode 100644 tests/keys/localhost.crt delete mode 100644 tests/keys/localhost.key diff --git a/.gitignore b/.gitignore index 83a2569dd..795e2605e 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ htmlcov/ .coverage.* .cache/ .mypy_cache/ +/tests/keys/localhost.crt +/tests/keys/localhost.key # Windows Thumbs.db diff --git a/conftest.py b/conftest.py index b39d644a5..95d4eaef6 100644 --- a/conftest.py +++ b/conftest.py @@ -2,6 +2,8 @@ from pathlib import Path import pytest +from tests.keys import generate_keys + def _py_files(folder): return (str(p) for p in Path(folder).rglob('*.py')) @@ -53,3 +55,7 @@ def reactor_pytest(request): def only_asyncio(request, reactor_pytest): if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio': pytest.skip('This test is only run with --reactor=asyncio') + + +# Generate localhost certificate files, needed by some tests +generate_keys() diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py new file mode 100644 index 000000000..da202be4d --- /dev/null +++ b/tests/keys/__init__.py @@ -0,0 +1,63 @@ +import os +from datetime import datetime, timedelta + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.hashes import SHA256 +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, +) +from cryptography.x509 import ( + CertificateBuilder, + DNSName, + Name, + NameAttribute, + random_serial_number, + SubjectAlternativeName, +) +from cryptography.x509.oid import NameOID + + +# https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate +def generate_keys(): + folder = os.path.dirname(__file__) + + key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + backend=default_backend(), + ) + with open(os.path.join(folder, 'localhost.key'), "wb") as f: + f.write( + key.private_bytes( + encoding=Encoding.PEM, + format=PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=NoEncryption(), + ) + ) + + subject = issuer = Name( + [ + NameAttribute(NameOID.COUNTRY_NAME, u"IE"), + NameAttribute(NameOID.ORGANIZATION_NAME, u"Scrapy"), + NameAttribute(NameOID.COMMON_NAME, u"localhost"), + ] + ) + cert = ( + CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(key.public_key()) + .serial_number(random_serial_number()) + .not_valid_before(datetime.utcnow()) + .not_valid_after(datetime.utcnow() + timedelta(days=10)) + .add_extension( + SubjectAlternativeName([DNSName(u"localhost")]), + critical=False, + ) + .sign(key, SHA256(), default_backend()) + ) + with open(os.path.join(folder, 'localhost.crt'), "wb") as f: + f.write(cert.public_bytes(Encoding.PEM)) diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt deleted file mode 100644 index 0cf5256d8..000000000 --- a/tests/keys/localhost.crt +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDRTCCAi2gAwIBAgIUGoISfeW3LwSWHC52ORXdZY9pNLswDQYJKoZIhvcNAQEL -BQAwMjELMAkGA1UEBhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9j -YWxob3N0MB4XDTIwMDYyODEyNTQxNVoXDTIxMDYyODEyNTQxNVowMjELMAkGA1UE -BhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjAN -BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvCLxfTEQuIdf8JhiHrbVkGHYrNSK -2XD2TCPaSIpJ2KKlFUrIz3A9tWlOfLnWabS5od89yOebhYj4DN/Qm2TViGg1mtWe -pD1K2YWd1Af+hhAw5D+TpW2RH9TVhX7Ey5osWcl+0uy+RlKZE8qum72xi1vxWOmH -wYw06iN8klQ3JfP2/eLRXBQjsh7WW0dbJ7yLvG6UFz1RbhFTtlxeIMenzNsHaMg7 -56Ru57/MMbaBwdBttXVzJDQ7imo8njuxDMszliC/QgIdBUBFzA2LB5qpr+v+laDN -cN9t9Q9stsu446dFnRoofxJjMFW7lLu6h/lwP5r0kfeUkMDhXJ4mb6KwfwIDAQAB -o1MwUTAdBgNVHQ4EFgQUVEdXn8ha2FA73zcy1Ia0FQMzMEYwHwYDVR0jBBgwFoAU -VEdXn8ha2FA73zcy1Ia0FQMzMEYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B -AQsFAAOCAQEAZpGBPsexMD+IwcMNIgc7FiaJsb8E30C9vWxgdnkpapi9zLJ4yiHQ -VxkV9RTezUEADkaDj+2qFveamWTzJLnphgaaUpVeMcYACPhRVOYXidNrZyTmHIsX -FwaTzAggW6CP7JxAcpxH0f9+NWFCZI36FihRdwuWyvrUl7rsXaexu0SOI/Ck0oWf -2IW+jo67TSmcbte+J8wq77DX32mVLb/2nqpItH4T2Di+XjVBARACVOSdgdlo7lZE -W8mSEXqP2BVx8JGG8X1znNLHcmjVj4EtkpH0wkYzpC4cvGkTsUcU7CU7ZyVUp+Bb -dPMVxyRKWfAjRJc8o5Ot1mgHrx5coOtzAA== ------END CERTIFICATE----- diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key deleted file mode 100644 index 8fc373bdd..000000000 --- a/tests/keys/localhost.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8IvF9MRC4h1/w -mGIettWQYdis1IrZcPZMI9pIiknYoqUVSsjPcD21aU58udZptLmh3z3I55uFiPgM -39CbZNWIaDWa1Z6kPUrZhZ3UB/6GEDDkP5OlbZEf1NWFfsTLmixZyX7S7L5GUpkT -yq6bvbGLW/FY6YfBjDTqI3ySVDcl8/b94tFcFCOyHtZbR1snvIu8bpQXPVFuEVO2 -XF4gx6fM2wdoyDvnpG7nv8wxtoHB0G21dXMkNDuKajyeO7EMyzOWIL9CAh0FQEXM -DYsHmqmv6/6VoM1w3231D2y2y7jjp0WdGih/EmMwVbuUu7qH+XA/mvSR95SQwOFc -niZvorB/AgMBAAECggEAHVpSVRb/pdqxNEeCH4qlHWa2uJhcpXpDYzPAzcqNpPgT -S5QkaoD3j8NDVKBl/I4O3FuJNzwzfo0VLmUJFgWQbzzbCDJGExfhArkfG8K3ilEi -X6ovrgK/PrklKzPRHncKbmPKnrwDH9OpQHZB8diRx81rhVTCModehh1NRUNQa2I1 -QzFC7uyXx3duoIsI5QXVeEGuwHZfqIY/z+9SscdVFL6elXTPFUzBzcmAqQgdgWKN -HXgX22LE0rAu8NnRvOZZWt4/nOjvlCFCPTB11NgthmKlVnsx4H7gpQ2OPh4bZ+0W -birVEtZ3E1jxoGvw1FzxyqqpGkcanRMa8QWzK4JwuQKBgQDrgclpkqZrgHB/TC1p -hLvsdflGI2SGs+c/mYR3GEjf0kJtI88WL5fj1QezdkDyOpwxFvnLslswfzdtzvis -vksGysV35vhMPQUcmWhvzA7Pdxdv4BZr+ckER0SAYBBxg9KYZyxewGb5XzB8Cz2o -8V+YpwrMAOYGuXHTfafv4CKlTQKBgQDMgetvV9/E3HNtKsATiPIwT3e1MzyPXigq -12NkHSZa6s4yqm/h/fSUn54sJbhx+OtRRhktOo0aB34tcogtrJyClvCPdRAP/4Qi -M43FjKo2cWiubWvtWlOZU04bpClG324q420rK7dCA2stID/Fa0sMQgAAyPH8TGMo -gbvyrk4W+wKBgQDMIOnYZTF0epaH8BponJFaqwMOhTzr+OGW4dTMebMotZG4EdK8 -kzIfW5XaOsSecKjTb+vCYGzkA1CjEEPBTwuu7nDstblAM5/Lozi/tmqb7sjUwrIM -kyxmVfONJjb6fV07lioCUtiui5B15DRkzBqlMRyNqLW43GJKA19d7rN4/QKBgCzy -kRBTu/bEjQn9T2H7w18i2CiXLkREaYeg91NVpMxutwsjspt0+YCA5H7He5ZxIycl -xPrP15tU8kKC3bNMMMny6sRc8j7R5fSuaAZ3OCHnIx7TJdlw9NbKHGyu0/Ojv87l -VWUbopd7sN6mK930CvaSuvVxNN5C27hXazuXW8ppAoGBANcWsenNKpCJgF0cNPHX -abPaWfcs5FKMNz8gEdGk3B1z/KBpYz59smPwurYVCXaWE6iv99sDOP7CVneF02sV -SqyNzVhcVSG788uB3CwnpEvm7ydoH89L5dvYekAHP8RJulhWCK45lXkHLiYGKvhv -PWuPk5VX+qF78JhUhPO3nfnu ------END PRIVATE KEY----- From 5e36f539e28631625862b84d8ca1c7c0134584b0 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 27 Aug 2020 15:12:22 +0530 Subject: [PATCH 281/568] chore: remove typing-extensions dependency --- scrapy/core/http2/protocol.py | 52 ++++++++++++++++----------- scrapy/core/http2/stream.py | 27 +++++++++++--- scrapy/core/http2/types.py | 67 ----------------------------------- setup.py | 1 - tox.ini | 1 - 5 files changed, 55 insertions(+), 93 deletions(-) delete mode 100644 scrapy/core/http2/types.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 6647ae0b7..0b872f6ba 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -3,7 +3,6 @@ import itertools import logging from collections import deque from ipaddress import IPv4Address, IPv6Address -from typing import Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -22,10 +21,10 @@ from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin from twisted.python.failure import Failure from twisted.web.client import URI +from typing import Dict, List, Optional, Union from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason -from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request from scrapy.settings import Settings from scrapy.spiders import Spider @@ -90,26 +89,39 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # all requests in a pool and send them as the connection is made self._pending_request_stream_pool: deque = deque() - # Counter to keep track of opened streams. This counter - # is used to make sure that not more than MAX_CONCURRENT_STREAMS - # streams are opened which leads to ProtocolError - # We use simple FIFO policy to handle pending requests - self._active_streams = 0 - - # Flag to keep track if settings were acknowledged by the remote - # This ensures that we have established a HTTP/2 connection - self._settings_acknowledged = False - # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure self._conn_lost_errors: List[BaseException] = [] - self.metadata: H2ConnectionMetadataDict = { + # Some meta data of this connection + # initialized when connection is successfully made + self.metadata: Dict = { + # Peer certificate instance 'certificate': None, + + # Address of the server we are connected to which + # is updated when HTTP/2 connection is made successfully 'ip_address': None, + + # URI of the peer HTTP/2 connection is made 'uri': uri, + + # Both ip_address and uri are used by the Stream before + # initiating the request to verify that the base address + + # Variables taken from Project Settings 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'), 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'), + + # Counter to keep track of opened streams. This counter + # is used to make sure that not more than MAX_CONCURRENT_STREAMS + # streams are opened which leads to ProtocolError + # We use simple FIFO policy to handle pending requests + 'active_streams': 0, + + # Flag to keep track if settings were acknowledged by the remote + # This ensures that we have established a HTTP/2 connection + 'settings_acknowledged': False, } @property @@ -118,7 +130,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection """ - return bool(self.transport.connected) and self._settings_acknowledged + return bool(self.transport.connected) and self.metadata['settings_acknowledged'] @property def allowed_max_concurrent_streams(self) -> int: @@ -139,10 +151,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """ while ( self._pending_request_stream_pool - and self._active_streams < self.allowed_max_concurrent_streams + and self.metadata['active_streams'] < self.allowed_max_concurrent_streams and self.h2_connected ): - self._active_streams += 1 + self.metadata['active_streams'] += 1 stream = self._pending_request_stream_pool.popleft() stream.initiate_request() self._write_to_transport() @@ -151,7 +163,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """Perform cleanup when a stream is closed """ stream = self.streams.pop(stream_id) - self._active_streams -= 1 + self.metadata['active_streams'] -= 1 self._send_pending_requests() return stream @@ -261,7 +273,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): if ( self.conn.open_outbound_streams > 0 or self.conn.open_inbound_streams > 0 - or self._active_streams > 0 + or self.metadata['active_streams'] > 0 ): error_code = ErrorCodes.PROTOCOL_ERROR else: @@ -295,7 +307,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): close_reason = StreamCloseReason.INACTIVE stream.close(close_reason, self._conn_lost_errors, from_protocol=True) - self._active_streams -= len(self.streams) + self.metadata['active_streams'] -= len(self.streams) self.streams.clear() self._pending_request_stream_pool.clear() self.conn.close_connection() @@ -338,7 +350,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: - self._settings_acknowledged = True + self.metadata['settings_acknowledged'] = True # Send off all the pending requests as now we have # established a proper HTTP/2 connection diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index df7470e11..e01e76ae5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,7 +1,6 @@ import logging from enum import Enum from io import BytesIO -from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes @@ -11,8 +10,9 @@ from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed +from typing import Dict +from typing import List, Optional, Tuple, TYPE_CHECKING -from scrapy.core.http2.types import H2ResponseDict, H2StreamMetadataDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes @@ -103,21 +103,40 @@ class Stream: self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) - self.metadata: H2StreamMetadataDict = { + # Metadata of an HTTP/2 connection stream + # initialized when stream is instantiated + self.metadata: Dict = { 'request_content_length': 0 if self._request.body is None else len(self._request.body), + + # Flag to keep track whether the stream has initiated the request 'request_sent': False, + + # Flag to track whether we have logged about exceeding download warnsize 'reached_warnsize': False, + + # Each time we send a data frame, we will decrease value by the amount send. 'remaining_content_length': 0 if self._request.body is None else len(self._request.body), + + # Flag to keep track whether client (self) have closed this stream 'stream_closed_local': False, + + # Flag to keep track whether the server has closed the stream 'stream_closed_server': False, } # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: H2ResponseDict = { + self._response: Dict = { + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. 'body': BytesIO(), + + # The amount of data received that counts against the + # flow control window 'flow_controlled_size': 0, + + # Headers received after sending the request 'headers': Headers({}), } diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py deleted file mode 100644 index ff8d94066..000000000 --- a/scrapy/core/http2/types.py +++ /dev/null @@ -1,67 +0,0 @@ -from io import BytesIO -from ipaddress import IPv4Address, IPv6Address -from typing import Union, Optional - -from twisted.internet.ssl import Certificate -from twisted.web.client import URI -# for python < 3.8 -- typing.TypedDict is undefined -from typing_extensions import TypedDict - -from scrapy.http.headers import Headers - - -class H2ConnectionMetadataDict(TypedDict): - """Some meta data of this connection - initialized when connection is successfully made - """ - certificate: Optional[Certificate] - - # Address of the server we are connected to which - # is updated when HTTP/2 connection is made successfully - ip_address: Optional[Union[IPv4Address, IPv6Address]] - - # URI of the peer HTTP/2 connection is made - uri: URI - - # Both ip_address and uri are used by the Stream before - # initiating the request to verify that the base address - - # Variables taken from Project Settings - default_download_maxsize: int - default_download_warnsize: int - - -class H2StreamMetadataDict(TypedDict): - """Metadata of an HTTP/2 connection stream - initialized when stream is instantiated - """ - - request_content_length: int - - # Flag to keep track whether the stream has initiated the request - request_sent: bool - - # Flag to track whether we have logged about exceeding download warnsize - reached_warnsize: bool - - # Each time we send a data frame, we will decrease value by the amount send. - remaining_content_length: int - - # Flag to keep track whether we have closed this stream - stream_closed_local: bool - - # Flag to keep track whether the server has closed the stream - stream_closed_server: bool - - -class H2ResponseDict(TypedDict): - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. - body: BytesIO - - # The amount of data received that counts against the flow control - # window - flow_controlled_size: int - - # Headers received after sending the request - headers: Headers diff --git a/setup.py b/setup.py index 39482d383..34af7ec67 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,6 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.2.0', - 'typing-extensions>=3.7.4', ] extras_require = {} diff --git a/tox.ini b/tox.ini index 0a88ed8af..78090d6de 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,6 @@ deps = queuelib==1.4.2 service_identity==16.0.0 Twisted[http2]==17.9.0 - typing-extensions==3.7.4 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt From 195f738bbaa7c090df795161390056c3c6c7f1f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 27 Aug 2020 12:43:43 +0200 Subject: [PATCH 282/568] Update Python version references after dropping support for 3.5 (#4742) * Update Python version references after dropping support for 3.5 * Remove outdated test * Undo change affecting collect_asyncgen * Undo change to be handled by #4743 * Remove unused import * Remove unused import * Update tests/requirements-py3.txt Co-authored-by: Mikhail Korobov Co-authored-by: Mikhail Korobov --- README.rst | 2 +- docs/intro/install.rst | 4 ++-- docs/intro/tutorial.rst | 2 -- docs/topics/coroutines.rst | 15 ++++----------- scrapy/__init__.py | 4 ++-- scrapy/utils/defer.py | 11 ++--------- scrapy/utils/gz.py | 7 +++---- setup.py | 3 +-- tests/requirements-py3.txt | 3 +-- tests/test_crawl.py | 4 ---- tests/test_item.py | 12 ------------ tests/test_proxy_connect.py | 11 ----------- tox.ini | 2 +- 13 files changed, 17 insertions(+), 63 deletions(-) diff --git a/README.rst b/README.rst index 0e3939e9b..a8f2ba52b 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ including a list of features. Requirements ============ -* Python 3.5.2+ +* Python 3.6+ * Works on Linux, Windows, macOS, BSD Install diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 6d65ae2ee..8b4240bf6 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,8 +9,8 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.5.2+, either the CPython implementation (default) or -the PyPy 5.9+ implementation (see :ref:`python:implementations`). +Scrapy requires Python 3.6+, either the CPython implementation (default) or +the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). Installing Scrapy diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index f96c78887..b3b8b4706 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -405,8 +405,6 @@ to get all of them: from sys import version_info -.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output") - Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary: diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a0952d323..3b1549bd3 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -17,19 +17,14 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.http.Request` callbacks. - The following are known caveats of the current implementation that we aim - to address in future versions of Scrapy: - - - The callback output is not processed until the whole callback finishes. + .. note:: The callback output is not processed until the whole callback + finishes. As a side effect, if the callback raises an exception, none of its output is processed. - - Because `asynchronous generators were introduced in Python 3.6`_, you - can only use ``yield`` if you are using Python 3.6 or later. - - If you need to output multiple items or requests and you are using - Python 3.5, return an iterable (e.g. a list) instead. + This is a known caveat of the current implementation that we aim to + address in a future version of Scrapy. - The :meth:`process_item` method of :ref:`item pipelines `. @@ -44,8 +39,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. -.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/ - Usage ===== diff --git a/scrapy/__init__.py b/scrapy/__init__.py index f0259a9b7..4326ca4aa 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version -if sys.version_info < (3, 5, 2): - print("Scrapy %s requires Python 3.5.2" % __version__) +if sys.version_info < (3, 6): + print("Scrapy %s requires Python 3.6+" % __version__) sys.exit(1) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index a3950db75..21ba02a0b 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -124,18 +124,11 @@ def iter_errback(iterable, errback, *a, **kw): errback(failure.Failure(), *a, **kw) -def _isfuture(o): - # workaround for Python before 3.5.3 not having asyncio.isfuture - if hasattr(asyncio, 'isfuture'): - return asyncio.isfuture(o) - return isinstance(o, asyncio.Future) - - def deferred_from_coro(o): """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, defer.Deferred): return o - if _isfuture(o) or inspect.isawaitable(o): + if asyncio.isfuture(o) or inspect.isawaitable(o): if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" @@ -167,7 +160,7 @@ def maybeDeferred_coro(f, *args, **kw): if isinstance(result, defer.Deferred): return result - elif _isfuture(result) or inspect.isawaitable(result): + elif asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) elif isinstance(result, failure.Failure): return defer.fail(result) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index fbd7bd18f..11d433cf5 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -6,11 +6,10 @@ import struct from scrapy.utils.decorators import deprecated -# - Python>=3.5 GzipFile's read() has issues returning leftover -# uncompressed data when input is corrupted -# (regression or bug-fix compared to Python 3.4) +# - GzipFile's read() has issues returning leftover uncompressed data when +# input is corrupted # - read1(), which fetches data before raising EOFError on next call -# works here but is only available from Python>=3.3 +# works here @deprecated('GzipFile.read1') def read1(gzf, size=-1): return gzf.read1(size) diff --git a/setup.py b/setup.py index 52a27c368..0c2281400 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,6 @@ setup( 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', @@ -92,7 +91,7 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=3.5.2', + python_requires='>=3.6', install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 44ddcded8..2247ed917 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -2,8 +2,7 @@ attrs dataclasses; python_version == '3.6' mitmproxy; python_version >= '3.7' -mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7' -mitmproxy < 4; python_version < '3.6' +mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 642c24651..c1b918baf 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,5 @@ import json import logging -import sys from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse @@ -405,7 +404,6 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("Got response 200", str(log)) self.assertIn({"foo": 42}, items) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): @@ -417,7 +415,6 @@ class CrawlSpiderTestCase(TestCase): itemcount = crawler.stats.get_value('item_scraped_count') self.assertEqual(itemcount, 1) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): @@ -437,7 +434,6 @@ class CrawlSpiderTestCase(TestCase): for i in range(10): self.assertIn({'foo': i}, items) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): diff --git a/tests/test_item.py b/tests/test_item.py index 66fa761f0..78d204e34 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,4 +1,3 @@ -import sys import unittest from unittest import mock from warnings import catch_warnings @@ -7,9 +6,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta -PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6) - - class ItemTest(unittest.TestCase): def assertSortedEqual(self, first, second, msg=None): @@ -280,14 +276,6 @@ class ItemMetaTest(unittest.TestCase): with mock.patch.object(base, '__new__', new_mock): class MyItem(Item): - if not PY36_PLUS: - # This attribute is an internal attribute in Python 3.6+ - # and must be propagated properly. See - # https://docs.python.org/3.6/reference/datamodel.html#creating-the-class-object - # In <3.6, we add a dummy attribute just to ensure the - # __new__ method propagates it correctly. - __classcell__ = object() - def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index a56e3c39a..6f70c4267 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -7,7 +7,6 @@ from subprocess import Popen, PIPE from urllib.parse import urlsplit, urlunsplit from unittest import skipIf -import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -58,8 +57,6 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) -@skipIf(sys.version_info < (3, 5, 4), - "requires mitmproxy < 3.0.0, which these tests do not support") @skipIf("pypy" in sys.executable, "mitmproxy does not support PyPy") @skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), @@ -88,14 +85,6 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, log) - @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info >= (3, 6)) - @defer.inlineCallbacks - def test_https_connect_tunnel_error(self): - crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl("https://localhost:99999/status?n=200") - self._assert_got_tunnel_error(log) - @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) diff --git a/tox.ini b/tox.ini index dec0d75e8..12e40295c 100644 --- a/tox.ini +++ b/tox.ini @@ -89,7 +89,7 @@ deps = basepython = python3 deps = {[pinned]deps} - # First lxml version that includes a Windows wheel for Python 3.5, so we do + # First lxml version that includes a Windows wheel for Python 3.6, so we do # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 From 3f0a677c0496da3d4d4294a182aa5e5b297a7cb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 27 Aug 2020 20:56:58 +0200 Subject: [PATCH 283/568] Cover version directive usage in the documentation policy (#4310) * Cover version directives in the documentation policy * Remove version directives in preparation for Scrapy 2.0 * Update the policy based on the deprecation policy * Only remove version directives after 3 years --- docs/contributing.rst | 11 ++++++++ docs/topics/api.rst | 2 -- docs/topics/autothrottle.rst | 2 -- docs/topics/benchmarking.rst | 2 -- docs/topics/commands.rst | 4 --- docs/topics/contracts.rst | 2 -- docs/topics/downloader-middleware.rst | 36 --------------------------- docs/topics/extensions.rst | 4 --- docs/topics/feed-exports.rst | 2 -- docs/topics/request-response.rst | 12 --------- docs/topics/settings.rst | 4 --- docs/topics/spider-middleware.rst | 6 ----- 12 files changed, 11 insertions(+), 76 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 525ad3497..675f55c38 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -199,6 +199,17 @@ In any case, if something is covered in a docstring, use the documentation instead of duplicating the docstring in files within the ``docs/`` directory. +Documentation updates that cover new or modified features must use Sphinx’s +:rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use +``VERSION`` as version, we will replace it with the actual version right before +the corresponding release. When we release a new major or minor version of +Scrapy, we remove these directives if they are older than 3 years. + +Documentation about deprecated features must be removed as those features are +deprecated, so that new readers do not run into it. New deprecations and +deprecation removals are documented in the :ref:`release notes `. + + Tests ===== diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 52509ffdf..445b2979f 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -4,8 +4,6 @@ Core API ======== -.. versionadded:: 0.15 - This section documents the Scrapy core API, and it's intended for developers of extensions and middlewares. diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 4317019fc..8e6aae65c 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -128,8 +128,6 @@ The maximum download delay (in seconds) to be set in case of high latencies. AUTOTHROTTLE_TARGET_CONCURRENCY ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 1.1 - Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 99469ebf1..b01a66188 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -4,8 +4,6 @@ Benchmarking ============ -.. versionadded:: 0.17 - Scrapy comes with a simple benchmarking suite that spawns a local HTTP server and crawls it at the maximum possible speed. The goal of this benchmarking is to get an idea of how Scrapy performs in your hardware, in order to have a diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 9638a2322..7de5e8121 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -6,8 +6,6 @@ Command line tool ================= -.. versionadded:: 0.10 - Scrapy is controlled through the ``scrapy`` command-line tool, to be referred here as the "Scrapy tool" to differentiate it from the sub-commands, which we just call "commands" or "Scrapy commands". @@ -566,8 +564,6 @@ and Platform info, which is useful for bug reports. bench ----- -.. versionadded:: 0.17 - * Syntax: ``scrapy bench`` * Requires project: *no* diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 430720fe3..e61421bf1 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -4,8 +4,6 @@ Spiders Contracts ================= -.. versionadded:: 0.15 - Testing spiders can get particularly annoying and while nothing prevents you from writing unit tests the task gets cumbersome quickly. Scrapy offers an integrated way of testing your spiders by the means of contracts. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 323e553e5..06e614941 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -217,8 +217,6 @@ The following settings can be used to configure the cookie middleware: Multiple cookie sessions per spider ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 0.15 - There is support for keeping multiple cookie sessions per spider by using the :reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar (session), but you can pass an identifier to use different ones. @@ -475,8 +473,6 @@ DBM storage backend .. class:: DbmCacheStorage - .. versionadded:: 0.13 - A DBM_ storage backend is also available for the HTTP cache middleware. By default, it uses the :mod:`dbm`, but you can change it with the @@ -549,15 +545,10 @@ settings: HTTPCACHE_ENABLED ^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.11 - Default: ``False`` Whether the HTTP cache will be enabled. -.. versionchanged:: 0.11 - Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache. - .. setting:: HTTPCACHE_EXPIRATION_SECS HTTPCACHE_EXPIRATION_SECS @@ -570,9 +561,6 @@ Expiration time for cached requests, in seconds. Cached requests older than this time will be re-downloaded. If zero, cached requests will never expire. -.. versionchanged:: 0.11 - Before 0.11, zero meant cached requests always expire. - .. setting:: HTTPCACHE_DIR HTTPCACHE_DIR @@ -589,8 +577,6 @@ project data dir. For more info see: :ref:`topics-project-structure`. HTTPCACHE_IGNORE_HTTP_CODES ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.10 - Default: ``[]`` Don't cache response with these HTTP codes. @@ -609,8 +595,6 @@ If enabled, requests not found in the cache will be ignored instead of downloade HTTPCACHE_IGNORE_SCHEMES ^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.10 - Default: ``['file']`` Don't cache responses with these URI schemes. @@ -629,8 +613,6 @@ The class which implements the cache storage backend. HTTPCACHE_DBM_MODULE ^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``'dbm'`` The database module to use in the :ref:`DBM storage backend @@ -641,8 +623,6 @@ The database module to use in the :ref:`DBM storage backend HTTPCACHE_POLICY ^^^^^^^^^^^^^^^^ -.. versionadded:: 0.18 - Default: ``'scrapy.extensions.httpcache.DummyPolicy'`` The class which implements the cache policy. @@ -652,8 +632,6 @@ The class which implements the cache policy. HTTPCACHE_GZIP ^^^^^^^^^^^^^^ -.. versionadded:: 1.0 - Default: ``False`` If enabled, will compress all cached data with gzip. @@ -664,8 +642,6 @@ This setting is specific to the Filesystem backend. HTTPCACHE_ALWAYS_STORE ^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 1.1 - Default: ``False`` If enabled, will cache pages unconditionally. @@ -684,8 +660,6 @@ responses you feed to the cache middleware. HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 1.1 - Default: ``[]`` List of Cache-Control directives in responses to be ignored. @@ -735,8 +709,6 @@ HttpProxyMiddleware .. module:: scrapy.downloadermiddlewares.httpproxy :synopsis: Http Proxy Middleware -.. versionadded:: 0.8 - .. reqmeta:: proxy .. class:: HttpProxyMiddleware @@ -817,8 +789,6 @@ RedirectMiddleware settings REDIRECT_ENABLED ^^^^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``True`` Whether the Redirect middleware will be enabled. @@ -860,8 +830,6 @@ MetaRefreshMiddleware settings METAREFRESH_ENABLED ^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.17 - Default: ``True`` Whether the Meta Refresh middleware will be enabled. @@ -924,8 +892,6 @@ RetryMiddleware Settings RETRY_ENABLED ^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``True`` Whether the Retry middleware will be enabled. @@ -1179,8 +1145,6 @@ AjaxCrawlMiddleware Settings AJAXCRAWL_ENABLED ^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.21 - Default: ``False`` Whether the AjaxCrawlMiddleware will be enabled. You may want to diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 0fc83e645..14096ada4 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -288,8 +288,6 @@ If zero (or non set), spiders won't be closed by number of passed items. CLOSESPIDER_PAGECOUNT """"""""""""""""""""" -.. versionadded:: 0.11 - Default: ``0`` An integer which specifies the maximum number of responses to crawl. If the spider @@ -302,8 +300,6 @@ number of crawled responses. CLOSESPIDER_ERRORCOUNT """""""""""""""""""""" -.. versionadded:: 0.11 - Default: ``0`` An integer which specifies the maximum number of errors to receive before diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index cd4f7cf29..9fb2189e8 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -4,8 +4,6 @@ Feed exports ============ -.. versionadded:: 0.10 - One of the most frequently required features when implementing scrapers is being able to store the scraped data properly and, quite often, that means generating an "export file" with the scraped data (commonly called "export diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d0136137f..30b1945d0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -553,18 +553,6 @@ fields with form data from :class:`Response` objects. The other parameters of this class method are passed directly to the :class:`FormRequest` ``__init__`` method. - .. versionadded:: 0.10.3 - The ``formname`` parameter. - - .. versionadded:: 0.17 - The ``formxpath`` parameter. - - .. versionadded:: 1.1.0 - The ``formcss`` parameter. - - .. versionadded:: 1.1.0 - The ``formid`` parameter. - Request usage examples ---------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2924c0566..d010a0023 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1076,8 +1076,6 @@ See :ref:`topics-extensions-ref-memusage`. MEMUSAGE_CHECK_INTERVAL_SECONDS ------------------------------- -.. versionadded:: 1.1 - Default: ``60.0`` Scope: ``scrapy.extensions.memusage`` @@ -1358,8 +1356,6 @@ The class that will be used for loading spiders, which must implement the SPIDER_LOADER_WARN_ONLY ----------------------- -.. versionadded:: 1.3.3 - Default: ``False`` By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c6cbdba76..fc114a63f 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -146,8 +146,6 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_start_requests(start_requests, spider) - .. versionadded:: 0.15 - This method is called with the start requests of the spider, and works similarly to the :meth:`process_spider_output` method, except that it doesn't have a response associated and must return only requests (not @@ -341,8 +339,6 @@ RefererMiddleware settings REFERER_ENABLED ^^^^^^^^^^^^^^^ -.. versionadded:: 0.15 - Default: ``True`` Whether to enable referer middleware. @@ -352,8 +348,6 @@ Whether to enable referer middleware. REFERRER_POLICY ^^^^^^^^^^^^^^^ -.. versionadded:: 1.4 - Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` .. reqmeta:: referrer_policy From 0e579182319504ba7bfd0c09333fe92f70c6d312 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Fri, 28 Aug 2020 13:58:32 +0200 Subject: [PATCH 284/568] test(Slot): cover __repr__ Issue: #4324 --- scrapy/core/downloader/__init__.py | 2 +- tests/test_core_downloader.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 tests/test_core_downloader.py diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 12a9db6dd..4f7ab594f 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -43,7 +43,7 @@ class Slot: cls_name = self.__class__.__name__ return (f"{cls_name}(concurrency={self.concurrency!r}, " f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r}") + f"randomize_delay={self.randomize_delay!r})") def __str__(self): return ( diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py new file mode 100644 index 000000000..113ea8f19 --- /dev/null +++ b/tests/test_core_downloader.py @@ -0,0 +1,10 @@ +from twisted.trial import unittest + +from scrapy.core.downloader import Slot + + +class SlotTest(unittest.TestCase): + + def test_repr(self): + slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) + self.assertEqual(repr(slot), 'Slot(concurrency=8, delay=0.10, randomize_delay=True)') From de640f41ecfa1a87b39d9e9268c396fc97353fc8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 28 Aug 2020 18:27:36 +0500 Subject: [PATCH 285/568] Merge back tests/py36/_test_crawl.py. --- conftest.py | 2 -- tests/py36/_test_crawl.py | 57 --------------------------------------- tests/spiders.py | 53 ++++++++++++++++++++++++++++++++++++ tests/test_crawl.py | 6 ++--- 4 files changed, 56 insertions(+), 62 deletions(-) delete mode 100644 tests/py36/_test_crawl.py diff --git a/conftest.py b/conftest.py index b39d644a5..be97b7714 100644 --- a/conftest.py +++ b/conftest.py @@ -14,8 +14,6 @@ collect_ignore = [ *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess *_py_files("tests/CrawlerRunner"), - # Py36-only parts of respective tests - *_py_files("tests/py36"), ] for line in open('tests/ignores.txt'): diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py deleted file mode 100644 index 162a53760..000000000 --- a/tests/py36/_test_crawl.py +++ /dev/null @@ -1,57 +0,0 @@ -import asyncio - -from scrapy import Request -from tests.spiders import SimpleSpider - - -class AsyncDefAsyncioGenSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen' - - async def parse(self, response): - await asyncio.sleep(0.2) - yield {'foo': 42} - self.logger.info("Got response %d" % response.status) - - -class AsyncDefAsyncioGenLoopSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen_loop' - - async def parse(self, response): - for i in range(10): - await asyncio.sleep(0.1) - yield {'foo': i} - self.logger.info("Got response %d" % response.status) - - -class AsyncDefAsyncioGenComplexSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen_complex' - initial_reqs = 4 - following_reqs = 3 - depth = 2 - - def _get_req(self, index, cb=None): - return Request(self.mockserver.url("/status?n=200&request=%d" % index), - meta={'index': index}, - dont_filter=True, - callback=cb) - - def start_requests(self): - for i in range(1, self.initial_reqs + 1): - yield self._get_req(i) - - async def parse(self, response): - index = response.meta['index'] - yield {'index': index} - if index < 10 ** self.depth: - for new_index in range(10 * index, 10 * index + self.following_reqs): - yield self._get_req(new_index) - yield self._get_req(index, cb=self.parse2) - await asyncio.sleep(0.1) - yield {'index': index + 5} - - async def parse2(self, response): - await asyncio.sleep(0.1) - yield {'index2': response.meta['index']} diff --git a/tests/spiders.py b/tests/spiders.py index 8a0ee44b7..8a85d2b51 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -148,6 +148,59 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): return reqs +class AsyncDefAsyncioGenSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen' + + async def parse(self, response): + await asyncio.sleep(0.2) + yield {'foo': 42} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenLoopSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_loop' + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {'foo': i} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenComplexSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_complex' + initial_reqs = 4 + following_reqs = 3 + depth = 2 + + def _get_req(self, index, cb=None): + return Request(self.mockserver.url("/status?n=200&request=%d" % index), + meta={'index': index}, + dont_filter=True, + callback=cb) + + def start_requests(self): + for i in range(1, self.initial_reqs + 1): + yield self._get_req(i) + + async def parse(self, response): + index = response.meta['index'] + yield {'index': index} + if index < 10 ** self.depth: + for new_index in range(10 * index, 10 * index + self.following_reqs): + yield self._get_req(new_index) + yield self._get_req(index, cb=self.parse2) + await asyncio.sleep(0.1) + yield {'index': index + 5} + + async def parse2(self, response): + await asyncio.sleep(0.1) + yield {'index2': response.meta['index']} + + class ItemSpider(FollowAllSpider): name = 'item' diff --git a/tests/test_crawl.py b/tests/test_crawl.py index c1b918baf..ba8c3fd3c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -19,6 +19,9 @@ from scrapy.http.response import Response from scrapy.utils.python import to_unicode from tests.mockserver import MockServer from tests.spiders import ( + AsyncDefAsyncioGenComplexSpider, + AsyncDefAsyncioGenLoopSpider, + AsyncDefAsyncioGenSpider, AsyncDefAsyncioReqsReturnSpider, AsyncDefAsyncioReturnSingleElementSpider, AsyncDefAsyncioReturnSpider, @@ -407,7 +410,6 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): - from tests.py36._test_crawl import AsyncDefAsyncioGenSpider crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) @@ -423,7 +425,6 @@ class CrawlSpiderTestCase(TestCase): def _on_item_scraped(item): items.append(item) - from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) with LogCapture() as log: @@ -442,7 +443,6 @@ class CrawlSpiderTestCase(TestCase): def _on_item_scraped(item): items.append(item) - from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) From ffdf6fe100cf0ac99a726eab9d927e65f4c01364 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Sat, 29 Aug 2020 07:21:48 +0200 Subject: [PATCH 286/568] use f-strings for the newly merged code from master Issue: #4324 --- tests/spiders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/spiders.py b/tests/spiders.py index 3c01c0217..106392ea6 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -177,7 +177,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): depth = 2 def _get_req(self, index, cb=None): - return Request(self.mockserver.url("/status?n=200&request=%d" % index), + return Request(self.mockserver.url(f"/status?n=200&request={index}"), meta={'index': index}, dont_filter=True, callback=cb) From a8aedbeb7c20c7e2ec041e49d1567a499cd3acdb Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 29 Aug 2020 12:12:18 +0530 Subject: [PATCH 287/568] chore: rearrange imports --- scrapy/core/http2/protocol.py | 2 +- scrapy/core/http2/stream.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 0b872f6ba..e32e2b6fe 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -3,6 +3,7 @@ import itertools import logging from collections import deque from ipaddress import IPv4Address, IPv6Address +from typing import Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -21,7 +22,6 @@ from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin from twisted.python.failure import Failure from twisted.web.client import URI -from typing import Dict, List, Optional, Union from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index e01e76ae5..ef90773b6 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -2,6 +2,7 @@ import logging from enum import Enum from io import BytesIO from urllib.parse import urlparse +from typing import Dict, List, Optional, Tuple, TYPE_CHECKING from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -10,8 +11,6 @@ from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from typing import Dict -from typing import List, Optional, Tuple, TYPE_CHECKING from scrapy.http import Request from scrapy.http.headers import Headers From 8123c427373778398036bb375bf03b3cd6b240de Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 28 Aug 2020 18:31:09 +0500 Subject: [PATCH 288/568] Simplify running spiders in CrawlSpiderTestCase. --- tests/test_crawl.py | 56 +++++++++++++++++---------------------------- 1 file changed, 21 insertions(+), 35 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index ba8c3fd3c..a8c9b6d7f 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -335,6 +335,19 @@ class CrawlSpiderTestCase(TestCase): def tearDown(self): self.mockserver.__exit__(None, None, None) + @defer.inlineCallbacks + def _run_spider(self, spider_cls): + items = [] + + def _on_item_scraped(item): + items.append(item) + + crawler = self.runner.create_crawler(spider_cls) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + return log, items, crawler.stats + @defer.inlineCallbacks def test_crawlspider_with_parse(self): self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver) @@ -379,15 +392,7 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse_items_list(self): - items = [] - - def _on_item_scraped(item): - items.append(item) - - crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSpider) - crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) self.assertIn("Got response 200", str(log)) self.assertIn({'id': 1}, items) self.assertIn({'id': 2}, items) @@ -410,27 +415,17 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): - crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) self.assertIn("Got response 200", str(log)) - itemcount = crawler.stats.get_value('item_scraped_count') + itemcount = stats.get_value('item_scraped_count') self.assertEqual(itemcount, 1) @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): - items = [] - - def _on_item_scraped(item): - items.append(item) - - crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider) - crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) self.assertIn("Got response 200", str(log)) - itemcount = crawler.stats.get_value('item_scraped_count') + itemcount = stats.get_value('item_scraped_count') self.assertEqual(itemcount, 10) for i in range(10): self.assertIn({'foo': i}, items) @@ -438,15 +433,8 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): - items = [] - - def _on_item_scraped(item): - items.append(item) - - crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider) - crawler.signals.connect(_on_item_scraped, signals.item_scraped) - yield crawler.crawl(mockserver=self.mockserver) - itemcount = crawler.stats.get_value('item_scraped_count') + _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) + itemcount = stats.get_value('item_scraped_count') self.assertEqual(itemcount, 156) # some random items for i in [1, 4, 21, 22, 207, 311]: @@ -457,9 +445,7 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): - crawler = self.runner.create_crawler(AsyncDefAsyncioReqsReturnSpider) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): self.assertIn("Got response 200, req_id %d" % req_id, str(log)) From 90ca9350f56c8c450c3385590d77390ad1d77365 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Sat, 29 Aug 2020 08:03:03 +0000 Subject: [PATCH 289/568] Update scrapy/commands/check.py Co-authored-by: Mikhail Korobov --- scrapy/commands/check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 7e848dc97..ae21d86e6 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult): plural = "s" if run != 1 else "" writeln(self.separator2) - writeln(f"Ran {run} contract{plural} in {stop - start:.3f}") + writeln(f"Ran {run} contract{plural} in {stop - start:.3f}s") writeln() infos = [] From a8e895e684184e967a751ea28a7102f21dd74834 Mon Sep 17 00:00:00 2001 From: maranqz Date: Sun, 30 Aug 2020 10:57:22 +0300 Subject: [PATCH 290/568] kwargs for Item exporters classes test docs --- docs/topics/feed-exports.rst | 5 +++++ scrapy/extensions/feedexport.py | 1 + scrapy/utils/conf.py | 1 + tests/test_feedexport.py | 37 +++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 9fb2189e8..e69a64195 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -303,6 +303,9 @@ For instance:: 'store_empty': False, 'fields': None, 'indent': 4, + 'item_export_kwargs': { + 'export_empty_fields': True, + }, }, '/home/user/documents/items.xml': { 'format': 'xml', @@ -332,6 +335,8 @@ as a fallback value if that key is not provided for a specific feed definition: - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. +- ``item_export_kwargs``: dict with kwargs for :ref:`Item exporters ` classes. + - ``overwrite``: whether to overwrite the file if it already exists (``True``) or append to its content (``False``). diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 980825499..f32d48fa5 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -360,6 +360,7 @@ class FeedExporter: fields_to_export=feed_options['fields'], encoding=feed_options['encoding'], indent=feed_options['indent'], + **feed_options['item_export_kwargs'], ) slot = _FeedSlot( file=file, diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 4e7a9967e..b904c4a03 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -121,6 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings): out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) + out.setdefault("item_export_kwargs", dict()) if settings["FEED_EXPORT_INDENT"] is None: out.setdefault("indent", None) else: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 840e0f87b..e88e4f5ce 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1252,6 +1252,43 @@ class FeedExportTest(FeedExportTestBase): for fmt in ['json', 'xml', 'csv']: self.assertIn(f'Error storing {fmt} feed (2 items)', str(log)) + @defer.inlineCallbacks + def test_extend_kwargs(self): + items = [{'foo': 'FOO', 'bar': 'BAR'}] + + expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8') + expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8') + test_cases = [ + # with title + { + 'options': { + 'format': 'csv', + 'item_export_kwargs': dict(include_headers_line=True), + }, + 'expected': expected_with_title_csv, + }, + # without title + { + 'options': { + 'format': 'csv', + 'item_export_kwargs': dict(include_headers_line=False), + }, + 'expected': expected_without_title_csv, + }, + ] + + for row in test_cases: + feed_options = row['options'] + settings = { + 'FEEDS': { + self._random_temp_filename(): feed_options, + }, + 'FEED_EXPORT_INDENT': None, + } + + data = yield self.exported_data(items, settings) + self.assertEqual(row['expected'], data[feed_options['format']]) + class BatchDeliveriesTest(FeedExportTestBase): __test__ = True From fc3c66ce950e945e7ac6e19fea31e8454147ac29 Mon Sep 17 00:00:00 2001 From: maranqz Date: Sun, 30 Aug 2020 11:44:48 +0300 Subject: [PATCH 291/568] fix tests: * FeedExportConfigTestCase.test_feed_complete_default_values_from_settings_empty * FeedExportConfigTestCase.test_feed_complete_default_values_from_settings_non_empty --- tests/test_utils_conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 061bc8c7c..dc2560add 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -176,6 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": (1, 2, 3, 4), "batch_item_count": 2, + "item_export_kwargs": dict(), }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -198,6 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": None, "batch_item_count": 2, + "item_export_kwargs": dict(), }) From 71d2c2f1a319b570eb2026707d49c4f62d59296e Mon Sep 17 00:00:00 2001 From: maranqz Date: Sun, 30 Aug 2020 12:43:44 +0300 Subject: [PATCH 292/568] improve view of dict --- tests/test_feedexport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e88e4f5ce..5738e36f1 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1263,7 +1263,7 @@ class FeedExportTest(FeedExportTestBase): { 'options': { 'format': 'csv', - 'item_export_kwargs': dict(include_headers_line=True), + 'item_export_kwargs': {'include_headers_line': True}, }, 'expected': expected_with_title_csv, }, @@ -1271,7 +1271,7 @@ class FeedExportTest(FeedExportTestBase): { 'options': { 'format': 'csv', - 'item_export_kwargs': dict(include_headers_line=False), + 'item_export_kwargs': {'include_headers_line': False}, }, 'expected': expected_without_title_csv, }, From eff33a2e7950b29fd69adc40b17b7fe9b0e637c2 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 30 Aug 2020 23:54:43 +0530 Subject: [PATCH 293/568] fix(h2): Mockserver test uses H2DownloadHandler --- tests/test_downloader_handlers.py | 7 ++++--- tests/test_downloader_handlers_http2.py | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e3777ee1d..c6f20cb50 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -601,6 +601,7 @@ class Https11CustomCiphers(unittest.TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" + settings_dict = None def setUp(self): self.mockserver = MockServer() @@ -611,7 +612,7 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_with_content_length(self): - crawler = get_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000})) @@ -620,7 +621,7 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download(self): - crawler = get_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) yield crawler.crawl(seed=Request(url=self.mockserver.url(''))) failure = crawler.spider.meta.get('failure') self.assertTrue(failure is None) @@ -629,7 +630,7 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_gzip_response(self): - crawler = get_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) body = b'1' * 100 # PayloadResource requires body length to be 100 request = Request(self.mockserver.url('/payload'), method='POST', body=body, meta={'download_maxsize': 50}) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 583dc1d17..253646040 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -110,6 +110,11 @@ class Https2CustomCiphers(Https11CustomCiphers): class Http2MockServerTestCase(Http11MockServerTestCase): """HTTP 2.0 test case with MockServer""" + settings_dict = { + 'DOWNLOAD_HANDLERS': { + 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler' + } + } class Https2ProxyTestCase(Http11ProxyTestCase): From a41c205928aa2aa86233de0cfb694b0c7ded2297 Mon Sep 17 00:00:00 2001 From: Jose Galdos Date: Fri, 21 Aug 2020 12:16:37 -0500 Subject: [PATCH 294/568] Update httpstatus documentation. --- docs/topics/spider-middleware.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c6cbdba76..28645fd53 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -255,7 +255,8 @@ this:: The ``handle_httpstatus_list`` key of :attr:`Request.meta ` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` -to ``True`` if you want to allow any response code for a request. +to ``True`` if you want to allow any response code for a request, and ``False`` to +disable the effects of the ``handle_httpstatus_all`` key. Keep in mind, however, that it's usually a bad idea to handle non-200 responses, unless you really know what you're doing. From d10464ca96a24d37c854992e2485d622e8eec2b6 Mon Sep 17 00:00:00 2001 From: Ilia Sergunin Date: Tue, 1 Sep 2020 10:13:40 +0300 Subject: [PATCH 295/568] Update docs/topics/feed-exports.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index e69a64195..1744cfd74 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -335,7 +335,7 @@ as a fallback value if that key is not provided for a specific feed definition: - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. -- ``item_export_kwargs``: dict with kwargs for :ref:`Item exporters ` classes. +- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. - ``overwrite``: whether to overwrite the file if it already exists (``True``) or append to its content (``False``). From 307e35c6641c0d9cf4b251996ebefd1347c2c1fd Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 1 Sep 2020 06:04:00 -0300 Subject: [PATCH 296/568] Improve check for invalid cookie in CookiesMiddleware (#4772) --- scrapy/downloadermiddlewares/cookies.py | 2 +- tests/test_downloadermiddleware_cookies.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index e2b7dd901..87f8152a4 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -74,7 +74,7 @@ class CookiesMiddleware: """ decoded = {} for key in ("name", "value", "path", "domain"): - if not cookie.get(key): + if cookie.get(key) is None: if key in ("name", "value"): msg = "Invalid cookie found in request {}: {} ('{}' is missing)" logger.warning(msg.format(request, cookie, key)) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 010577415..a3de307ee 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -322,6 +322,9 @@ class CookiesMiddlewareTest(TestCase): cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}] req2 = Request('http://example.org/2', cookies=cookies2) assert self.mw.process_request(req2, self.spider) is None + cookies3 = [{'name': 'foo', 'value': None}, {'name': 'key', 'value': ''}] + req3 = Request('http://example.org/3', cookies=cookies3) + assert self.mw.process_request(req3, self.spider) is None lc.check( ("scrapy.downloadermiddlewares.cookies", "WARNING", @@ -331,6 +334,11 @@ class CookiesMiddlewareTest(TestCase): "WARNING", "Invalid cookie found in request :" " {'name': 'foo'} ('value' is missing)"), + ("scrapy.downloadermiddlewares.cookies", + "WARNING", + "Invalid cookie found in request :" + " {'name': 'foo', 'value': None} ('value' is missing)"), ) self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1') self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2') + self.assertCookieValEqual(req3.headers['Cookie'], 'key=') From ddc26f3f8fdf766587eda55dfaf60b524eb89e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Sep 2020 11:26:07 +0200 Subject: [PATCH 297/568] Revert Travis CI changes --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d9a8e512a..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ dist: xenial branches: only: - master - - http2 # ToDo: Remove once merged into master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: From 959222df7e80334810636166a8adb60ef35728de Mon Sep 17 00:00:00 2001 From: drs-11 Date: Sat, 5 Sep 2020 21:32:05 +0530 Subject: [PATCH 298/568] check for unparseable no_proxy values --- scrapy/downloadermiddlewares/httpproxy.py | 7 ++++++- tests/test_downloadermiddleware_httpproxy.py | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 04da11311..d2665b655 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -13,7 +13,12 @@ class HttpProxyMiddleware: self.auth_encoding = auth_encoding self.proxies = {} for type_, url in getproxies().items(): - self.proxies[type_] = self._get_proxy(url, type_) + try: + self.proxies[type_] = self._get_proxy(url, type_) + # some values such as '/var/run/docker.sock' can't be parsed + # by _parse_proxy and as such should be skipped + except ValueError: + continue @classmethod def from_crawler(cls, crawler): diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 351631eb8..81d6cc335 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -123,7 +123,11 @@ class TestHttpProxyMiddleware(TestCase): def test_no_proxy(self): os.environ['http_proxy'] = 'https://proxy.for.http:3128' + os.environ['no_proxy'] = '/var/run/docker.sock' mw = HttpProxyMiddleware() + # '/var/run/docker.sock' may be used by the user for + # no_proxy value but is not parseable and should be skipped + assert 'no' not in mw.proxies os.environ['no_proxy'] = '*' req = Request('http://noproxy.com') From 7a83474cc534642e8daf581d4b427408343377f2 Mon Sep 17 00:00:00 2001 From: KAILASA's Sri Nithya Priyeshananda <68758690+sripriyesha@users.noreply.github.com> Date: Tue, 8 Sep 2020 17:16:31 +0200 Subject: [PATCH 299/568] add mention of FTP server storage in media storage intro At the beginning of this doc, in "Specifying where to store the media" feature details, FTP server storage mention was missing --- docs/topics/media-pipeline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 1f995ce14..c2255ea79 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -15,7 +15,7 @@ typically you'll either use the Files Pipeline or the Images Pipeline. Both pipelines implement these features: * Avoid re-downloading media that was downloaded recently -* Specifying where to store the media (filesystem directory, Amazon S3 bucket, +* Specifying where to store the media (filesystem directory, FTP server, Amazon S3 bucket, Google Cloud Storage bucket) The Images Pipeline has a few extra functions for processing images: From 82ba7c8b529e004a62db4681f16a482ae4e769f1 Mon Sep 17 00:00:00 2001 From: drs-11 Date: Thu, 10 Sep 2020 20:56:39 +0530 Subject: [PATCH 300/568] created separate test for invalid no-proxy values --- tests/test_downloadermiddleware_httpproxy.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 81d6cc335..7c97bf32a 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -123,11 +123,7 @@ class TestHttpProxyMiddleware(TestCase): def test_no_proxy(self): os.environ['http_proxy'] = 'https://proxy.for.http:3128' - os.environ['no_proxy'] = '/var/run/docker.sock' mw = HttpProxyMiddleware() - # '/var/run/docker.sock' may be used by the user for - # no_proxy value but is not parseable and should be skipped - assert 'no' not in mw.proxies os.environ['no_proxy'] = '*' req = Request('http://noproxy.com') @@ -149,3 +145,10 @@ class TestHttpProxyMiddleware(TestCase): req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'}) assert mw.process_request(req, spider) is None self.assertEqual(req.meta, {'proxy': 'http://proxy.com'}) + + def test_no_proxy_invalid_values(self): + os.environ['no_proxy'] = '/var/run/docker.sock' + mw = HttpProxyMiddleware() + # '/var/run/docker.sock' may be used by the user for + # no_proxy value but is not parseable and should be skipped + assert 'no' not in mw.proxies From 4d6359df2dfa3d561088faf097f6abe3f850a4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 11 Sep 2020 13:51:05 +0200 Subject: [PATCH 301/568] Mark HTTP/2 as experimental --- docs/topics/asyncio.rst | 11 ++++++----- docs/topics/commands.rst | 2 -- docs/topics/settings.rst | 4 ++++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index bfb430d52..c04044e8f 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -4,13 +4,14 @@ asyncio .. versionadded:: 2.0 -Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio -reactor `, you may use :mod:`asyncio` and +Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the +asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy - versions may introduce related changes without a deprecation - period or warning. +.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet + recommended for production environments. Future Scrapy versions + may introduce related changes without a deprecation period or + warning. .. _install-asyncio: diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 7de5e8121..eef6b36ff 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -598,8 +598,6 @@ Example: Register commands via setup.py entry points ------------------------------------------- -.. note:: This is an experimental feature, use with caution. - You can also add Scrapy commands from an external library by adding a ``scrapy.commands`` section in the entry points of the library ``setup.py`` file. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8794e0259..218b23c87 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -678,6 +678,10 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update Scrapy currently does not support HTTP/2 Cleartext (h2c) since none of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_). +.. warning:: HTTP/2 support in Scrapy is experimental, and not yet recommended + for production environments. Future Scrapy versions may introduce + related changes without a deprecation period or warning. + .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. setting:: DOWNLOAD_TIMEOUT From 6e8d20a07a8c1a1f3ad7b3c3d74b7d37f2b47327 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 16 Sep 2020 04:57:07 -0300 Subject: [PATCH 302/568] HTTP/2: add some type hints (#4785) --- scrapy/core/downloader/handlers/http2.py | 24 +++++++++++++-------- scrapy/core/http2/agent.py | 27 ++++++++++++++---------- scrapy/core/http2/protocol.py | 15 ++++++++----- scrapy/core/http2/stream.py | 7 +++--- setup.cfg | 6 ------ tests/test_downloader_handlers.py | 19 +++++++++-------- tests/test_http2_client_protocol.py | 2 +- 7 files changed, 56 insertions(+), 44 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 4be888bda..e97c31e90 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,8 +1,9 @@ import warnings from time import time -from typing import Optional +from typing import Optional, Type, TypeVar from urllib.parse import urldefrag +from twisted.internet.base import DelayedCall from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI @@ -10,14 +11,18 @@ from twisted.web.client import URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent +from scrapy.crawler import Crawler from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes +H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler") + + class H2DownloadHandler: - def __init__(self, settings: Settings, crawler=None): + def __init__(self, settings: Settings, crawler: Optional[Crawler] = None): self._crawler = crawler from twisted.internet import reactor @@ -25,14 +30,14 @@ class H2DownloadHandler: self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass: return cls(crawler.settings, crawler) def download_request(self, request: Request, spider: Spider) -> Deferred: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, - crawler=self._crawler + crawler=self._crawler, ) return agent.download_request(request, spider) @@ -47,8 +52,9 @@ class ScrapyH2Agent: def __init__( self, context_factory, pool: H2ConnectionPool, - connect_timeout=10, bind_address: Optional[bytes] = None, - crawler=None + connect_timeout: int = 10, + bind_address: Optional[bytes] = None, + crawler: Optional[Crawler] = None, ) -> None: self._context_factory = context_factory self._connect_timeout = connect_timeout @@ -80,7 +86,7 @@ class ScrapyH2Agent: proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')), connect_timeout=timeout, bind_address=bind_address, - pool=self._pool + pool=self._pool, ) return self._Agent( @@ -88,7 +94,7 @@ class ScrapyH2Agent: context_factory=self._context_factory, connect_timeout=timeout, bind_address=bind_address, - pool=self._pool + pool=self._pool, ) def download_request(self, request: Request, spider: Spider) -> Deferred: @@ -110,7 +116,7 @@ class ScrapyH2Agent: return response @staticmethod - def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl) -> Response: + def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response: if timeout_cl.active(): timeout_cl.cancel() return response diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index d950c6cfb..a142fa210 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,5 +1,5 @@ from collections import deque -from typing import Deque, Dict, List, Tuple, Optional +from typing import Deque, Dict, List, Optional, Tuple from twisted.internet import defer from twisted.internet.base import ReactorBase @@ -95,16 +95,18 @@ class H2ConnectionPool: class H2Agent: def __init__( - self, reactor: ReactorBase, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + self, + reactor: ReactorBase, + pool: H2ConnectionPool, + context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, + bind_address: Optional[bytes] = None, ) -> None: self._reactor = reactor self._pool = pool self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2']) self.endpoint_factory = _StandardEndpointFactory( - self._reactor, self._context_factory, - connect_timeout, bind_address + self._reactor, self._context_factory, connect_timeout, bind_address ) def get_endpoint(self, uri: URI): @@ -132,17 +134,20 @@ class H2Agent: class ScrapyProxyH2Agent(H2Agent): def __init__( - self, reactor: ReactorBase, - proxy_uri: URI, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + self, + reactor: ReactorBase, + proxy_uri: URI, + pool: H2ConnectionPool, + context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, + bind_address: Optional[bytes] = None, ) -> None: super(ScrapyProxyH2Agent, self).__init__( reactor=reactor, pool=pool, context_factory=context_factory, connect_timeout=connect_timeout, - bind_address=bind_address + bind_address=bind_address, ) self._proxy_uri = proxy_uri diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index e32e2b6fe..9d499596c 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -29,6 +29,7 @@ from scrapy.http import Request from scrapy.settings import Settings from scrapy.spiders import Spider + logger = logging.getLogger(__name__) @@ -42,7 +43,11 @@ class InvalidNegotiatedProtocol(H2Error): class RemoteTerminatedConnection(H2Error): - def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], event: ConnectionTerminated): + def __init__( + self, + remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], + event: ConnectionTerminated, + ) -> None: self.remote_ip_address = remote_ip_address self.terminate_event = event @@ -51,7 +56,7 @@ class RemoteTerminatedConnection(H2Error): class MethodNotAllowed405(H2Error): - def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]): + def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None: self.remote_ip_address = remote_ip_address def __str__(self) -> str: @@ -220,13 +225,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.conn.initiate_connection() self._write_to_transport() - def _lose_connection_with_error(self, errors: List[BaseException]): + def _lose_connection_with_error(self, errors: List[BaseException]) -> None: """Helper function to lose the connection with the error sent as a reason""" self._conn_lost_errors += errors self.transport.loseConnection() - def handshakeCompleted(self): + def handshakeCompleted(self) -> None: """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" negotiated_protocol = self.transport.negotiatedProtocol @@ -263,7 +268,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): finally: self._write_to_transport() - def timeoutConnection(self): + def timeoutConnection(self) -> None: """Called when the connection times out. We lose the connection with TimeoutError""" diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index ef90773b6..3ae2e8db8 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -19,6 +19,7 @@ from scrapy.responsetypes import responsetypes if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol + logger = logging.getLogger(__name__) @@ -27,7 +28,7 @@ class InactiveStreamClosed(ConnectionClosed): of the stream. This happens when a stream is waiting for other streams to close and connection is lost.""" - def __init__(self, request: Request): + def __init__(self, request: Request) -> None: self.request = request def __str__(self) -> str: @@ -139,7 +140,7 @@ class Stream: 'headers': Headers({}), } - def _cancel(_): + def _cancel(_) -> None: # Close this stream as gracefully as possible # If the associated request is initiated we reset this stream # else we directly call close() method @@ -360,7 +361,7 @@ class Stream: self, reason: StreamCloseReason, errors: Optional[List[BaseException]] = None, - from_protocol: bool = False + from_protocol: bool = False, ) -> None: """Based on the reason sent we will handle each case. """ diff --git a/setup.cfg b/setup.cfg index 94c1e7b89..8101443e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -82,9 +82,6 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_downloader_handlers_http2] -ignore_errors = True - [mypy-tests.test_engine] ignore_errors = True @@ -94,9 +91,6 @@ ignore_errors = True [mypy-tests.test_http_request] ignore_errors = True -[mypy-tests.test_http2_client_protocol] -ignore_errors = True - [mypy-tests.test_linkextractors] ignore_errors = True diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 20e31f7f7..5b4a2d270 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -2,6 +2,7 @@ import contextlib import os import shutil import tempfile +from typing import Optional, Type from unittest import mock from testfixtures import LogCapture @@ -206,7 +207,7 @@ class LargeChunkedFileResource(resource.Resource): class HttpTestCase(unittest.TestCase): scheme = 'http' - download_handler_cls = HTTPDownloadHandler + download_handler_cls: Type = HTTPDownloadHandler # only used for HTTPS tests keyfile = 'keys/localhost.key' @@ -365,7 +366,7 @@ class HttpTestCase(unittest.TestCase): class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" - download_handler_cls = HTTP10DownloadHandler + download_handler_cls: Type = HTTP10DownloadHandler class Https10TestCase(Http10TestCase): @@ -374,7 +375,7 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler def test_download_without_maxsize_limit(self): request = Request(self.getURL('file')) @@ -561,7 +562,7 @@ class Https11InvalidDNSPattern(Https11TestCase): class Https11CustomCiphers(unittest.TestCase): scheme = 'https' - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler keyfile = 'keys/localhost.key' certfile = 'keys/localhost.crt' @@ -601,7 +602,7 @@ class Https11CustomCiphers(unittest.TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" - settings_dict = None + settings_dict: Optional[dict] = None def setUp(self): self.mockserver = MockServer() @@ -668,7 +669,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): - download_handler_cls = HTTPDownloadHandler + download_handler_cls: Type = HTTPDownloadHandler expected_http_proxy_request_body = b'http://example.com' def setUp(self): @@ -721,14 +722,14 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): - download_handler_cls = HTTP10DownloadHandler + download_handler_cls: Type = HTTP10DownloadHandler def test_download_with_proxy_https_noconnect(self): raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler') class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -776,7 +777,7 @@ class S3AnonTestCase(unittest.TestCase): class S3TestCase(unittest.TestCase): - download_handler_cls = S3DownloadHandler + download_handler_cls: Type = S3DownloadHandler # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 4926ada14..d9ab553f0 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -45,7 +45,7 @@ def make_html_body(val): class DummySpider(Spider): name = 'dummy' - start_urls = [] + start_urls: list = [] def parse(self, response): print(response) From 5e997587d9b13344a0afa9bb4cf781829a66ce23 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Sun, 20 Sep 2020 18:06:46 +0500 Subject: [PATCH 303/568] Remove dead boto2 code, deprecate is_botocore() (#4776) --- scrapy/core/downloader/handlers/s3.py | 55 ++++----------- scrapy/extensions/feedexport.py | 35 ++++------ scrapy/pipelines/files.py | 97 +++++++++------------------ scrapy/utils/boto.py | 23 ++++++- scrapy/utils/test.py | 29 +++----- tests/test_feedexport.py | 69 ++----------------- tests/test_pipeline_files.py | 15 ++--- 7 files changed, 96 insertions(+), 227 deletions(-) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 0ef977893..1966570d4 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,41 +2,20 @@ from urllib.parse import unquote from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured -from scrapy.utils.boto import is_botocore +from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import create_instance -def _get_boto_connection(): - from boto.s3.connection import S3Connection - - class _v19_S3Connection(S3Connection): - """A dummy S3Connection wrapper that doesn't do any synchronous download""" - def _mexe(self, method, bucket, key, headers, *args, **kwargs): - return headers - - class _v20_S3Connection(S3Connection): - """A dummy S3Connection wrapper that doesn't do any synchronous download""" - def _mexe(self, http_request, *args, **kwargs): - http_request.authorize(connection=self) - return http_request.headers - - try: - import boto.auth # noqa: F401 - except ImportError: - _S3Connection = _v19_S3Connection - else: - _S3Connection = _v20_S3Connection - - return _S3Connection - - class S3DownloadHandler: def __init__(self, settings, *, crawler=None, aws_access_key_id=None, aws_secret_access_key=None, httpdownloadhandler=HTTPDownloadHandler, **kw): + if not is_botocore_available(): + raise NotConfigured('missing botocore library') + if not aws_access_key_id: aws_access_key_id = settings['AWS_ACCESS_KEY_ID'] if not aws_secret_access_key: @@ -51,23 +30,15 @@ class S3DownloadHandler: self.anon = kw.get('anon') self._signer = None - if is_botocore(): - import botocore.auth - import botocore.credentials - kw.pop('anon', None) - if kw: - raise TypeError(f'Unexpected keyword arguments: {kw}') - if not self.anon: - SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] - self._signer = SignerCls(botocore.credentials.Credentials( - aws_access_key_id, aws_secret_access_key)) - else: - _S3Connection = _get_boto_connection() - try: - self.conn = _S3Connection( - aws_access_key_id, aws_secret_access_key, **kw) - except Exception as ex: - raise NotConfigured(str(ex)) + import botocore.auth + import botocore.credentials + kw.pop('anon', None) + if kw: + raise TypeError(f'Unexpected keyword arguments: {kw}') + if not self.anon: + SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] + self._signer = SignerCls(botocore.credentials.Credentials( + aws_access_key_id, aws_secret_access_key)) _http_handler = create_instance( objcls=httpdownloadhandler, diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 980825499..9f712285f 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -19,7 +19,7 @@ from zope.interface import implementer, Interface from scrapy import signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.utils.boto import is_botocore +from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info @@ -120,22 +120,19 @@ class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None, acl=None, *, feed_options=None): + if not is_botocore_available(): + raise NotConfigured('missing botocore library') u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key self.secret_key = u.password or secret_key - self.is_botocore = is_botocore() self.keyname = u.path[1:] # remove first "/" self.acl = acl - if self.is_botocore: - import botocore.session - session = botocore.session.get_session() - self.s3_client = session.create_client( - 's3', aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key) - else: - import boto - self.connect_s3 = boto.connect_s3 + import botocore.session + session = botocore.session.get_session() + self.s3_client = session.create_client( + 's3', aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key) if feed_options and feed_options.get('overwrite', True) is False: logger.warning('S3 does not support appending to files. To ' 'suppress this warning, remove the overwrite ' @@ -154,18 +151,10 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file): file.seek(0) - if self.is_botocore: - kwargs = {'ACL': self.acl} if self.acl else {} - self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file, - **kwargs) - else: - conn = self.connect_s3(self.access_key, self.secret_key) - bucket = conn.get_bucket(self.bucketname, validate=False) - key = bucket.new_key(self.keyname) - kwargs = {'policy': self.acl} if self.acl else {} - key.set_contents_from_file(file, **kwargs) - key.close() + kwargs = {'ACL': self.acl} if self.acl else {} + self.s3_client.put_object( + Bucket=self.bucketname, Key=self.keyname, Body=file, + **kwargs) file.close() diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 99a72aa70..13ecd4e6c 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -11,7 +11,6 @@ import os import time from collections import defaultdict from contextlib import suppress -from email.utils import mktime_tz, parsedate_tz from ftplib import FTP from io import BytesIO from urllib.parse import urlparse @@ -23,7 +22,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings -from scrapy.utils.boto import is_botocore +from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaselessDict from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info @@ -91,86 +90,54 @@ class S3FilesStore: } def __init__(self, uri): - self.is_botocore = is_botocore() - if self.is_botocore: - import botocore.session - session = botocore.session.get_session() - self.s3_client = session.create_client( - 's3', - aws_access_key_id=self.AWS_ACCESS_KEY_ID, - aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, - endpoint_url=self.AWS_ENDPOINT_URL, - region_name=self.AWS_REGION_NAME, - use_ssl=self.AWS_USE_SSL, - verify=self.AWS_VERIFY - ) - else: - from boto.s3.connection import S3Connection - self.S3Connection = S3Connection + if not is_botocore_available(): + raise NotConfigured('missing botocore library') + import botocore.session + session = botocore.session.get_session() + self.s3_client = session.create_client( + 's3', + aws_access_key_id=self.AWS_ACCESS_KEY_ID, + aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, + endpoint_url=self.AWS_ENDPOINT_URL, + region_name=self.AWS_REGION_NAME, + use_ssl=self.AWS_USE_SSL, + verify=self.AWS_VERIFY + ) if not uri.startswith("s3://"): raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") self.bucket, self.prefix = uri[5:].split('/', 1) def stat_file(self, path, info): def _onsuccess(boto_key): - if self.is_botocore: - checksum = boto_key['ETag'].strip('"') - last_modified = boto_key['LastModified'] - modified_stamp = time.mktime(last_modified.timetuple()) - else: - checksum = boto_key.etag.strip('"') - last_modified = boto_key.last_modified - modified_tuple = parsedate_tz(last_modified) - modified_stamp = int(mktime_tz(modified_tuple)) + checksum = boto_key['ETag'].strip('"') + last_modified = boto_key['LastModified'] + modified_stamp = time.mktime(last_modified.timetuple()) return {'checksum': checksum, 'last_modified': modified_stamp} return self._get_boto_key(path).addCallback(_onsuccess) - def _get_boto_bucket(self): - # disable ssl (is_secure=False) because of this python bug: - # https://bugs.python.org/issue5103 - c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False) - return c.get_bucket(self.bucket, validate=False) - def _get_boto_key(self, path): key_name = f'{self.prefix}{path}' - if self.is_botocore: - return threads.deferToThread( - self.s3_client.head_object, - Bucket=self.bucket, - Key=key_name) - else: - b = self._get_boto_bucket() - return threads.deferToThread(b.get_key, key_name) + return threads.deferToThread( + self.s3_client.head_object, + Bucket=self.bucket, + Key=key_name) def persist_file(self, path, buf, info, meta=None, headers=None): """Upload file to S3 storage""" key_name = f'{self.prefix}{path}' buf.seek(0) - if self.is_botocore: - extra = self._headers_to_botocore_kwargs(self.HEADERS) - if headers: - extra.update(self._headers_to_botocore_kwargs(headers)) - return threads.deferToThread( - self.s3_client.put_object, - Bucket=self.bucket, - Key=key_name, - Body=buf, - Metadata={k: str(v) for k, v in (meta or {}).items()}, - ACL=self.POLICY, - **extra) - else: - b = self._get_boto_bucket() - k = b.new_key(key_name) - if meta: - for metakey, metavalue in meta.items(): - k.set_metadata(metakey, str(metavalue)) - h = self.HEADERS.copy() - if headers: - h.update(headers) - return threads.deferToThread( - k.set_contents_from_string, buf.getvalue(), - headers=h, policy=self.POLICY) + extra = self._headers_to_botocore_kwargs(self.HEADERS) + if headers: + extra.update(self._headers_to_botocore_kwargs(headers)) + return threads.deferToThread( + self.s3_client.put_object, + Bucket=self.bucket, + Key=key_name, + Body=buf, + Metadata={k: str(v) for k, v in (meta or {}).items()}, + ACL=self.POLICY, + **extra) def _headers_to_botocore_kwargs(self, headers): """ Convert headers to botocore keyword agruments. diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 12321caa5..3374c57c7 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,11 +1,32 @@ """Boto/botocore helpers""" +import warnings -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning def is_botocore(): + """ Returns True if botocore is available, otherwise raises NotConfigured. Never returns False. + + Previously, when boto was supported in addition to botocore, this returned False if boto was available + but botocore wasn't. + """ + message = ( + 'is_botocore() is deprecated and always returns True or raises an Exception, ' + 'so it cannot be used for checking if boto is available instead of botocore. ' + 'You can use scrapy.utils.boto.is_botocore_available() to check if botocore ' + 'is available.' + ) + warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) try: import botocore # noqa: F401 return True except ImportError: raise NotConfigured('missing botocore library') + + +def is_botocore_available(): + try: + import botocore # noqa: F401 + return True + except ImportError: + return False diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index f54942ffb..94d0ae2d3 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -10,8 +10,7 @@ from unittest import mock from importlib import import_module from twisted.trial.unittest import SkipTest -from scrapy.exceptions import NotConfigured -from scrapy.utils.boto import is_botocore +from scrapy.utils.boto import is_botocore_available def assert_aws_environ(): @@ -29,29 +28,19 @@ def assert_gcs_environ(): def skip_if_no_boto(): - try: - is_botocore() - except NotConfigured as e: - raise SkipTest(e) + if not is_botocore_available(): + raise SkipTest('missing botocore library') def get_s3_content_and_delete(bucket, path, with_key=False): """ Get content from s3 key, and delete key afterwards. """ - if is_botocore(): - import botocore.session - session = botocore.session.get_session() - client = session.create_client('s3') - key = client.get_object(Bucket=bucket, Key=path) - content = key['Body'].read() - client.delete_object(Bucket=bucket, Key=path) - else: - import boto - # assuming boto=2.2.2 - bucket = boto.connect_s3().get_bucket(bucket, validate=False) - key = bucket.get_key(path) - content = key.get_contents_as_string() - bucket.delete_key(path) + import botocore.session + session = botocore.session.get_session() + client = session.create_client('s3') + key = client.get_object(Bucket=bucket, Key=path) + content = key['Body'].read() + client.delete_object(Bucket=bucket, Key=path) return (content, key) if with_key else content diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 840e0f87b..33ac51712 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -45,6 +45,7 @@ from scrapy.utils.test import ( get_s3_content_and_delete, get_crawler, mock_google_cloud_storage, + skip_if_no_boto, ) from tests.mockserver import MockFTPServer, MockServer @@ -227,10 +228,7 @@ class BlockingFeedStorageTest(unittest.TestCase): class S3FeedStorageTest(unittest.TestCase): def test_parse_credentials(self): - try: - import botocore # noqa: F401 - except ImportError: - raise unittest.SkipTest("S3FeedStorage requires botocore") + skip_if_no_boto() aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} crawler = get_crawler(settings_dict=aws_credentials) @@ -324,11 +322,7 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_without_acl(self): - try: - import botocore # noqa: F401 - except ImportError: - raise unittest.SkipTest('botocore is required') - + skip_if_no_boto() storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -344,11 +338,7 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_with_acl(self): - try: - import botocore # noqa: F401 - except ImportError: - raise unittest.SkipTest('botocore is required') - + skip_if_no_boto() storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -366,57 +356,6 @@ class S3FeedStorageTest(unittest.TestCase): 'custom-acl' ) - @defer.inlineCallbacks - def test_store_not_botocore_without_acl(self): - storage = S3FeedStorage( - 's3://mybucket/export.csv', - 'access_key', - 'secret_key', - ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') - self.assertEqual(storage.acl, None) - - storage.is_botocore = False - storage.connect_s3 = mock.MagicMock() - self.assertFalse(storage.is_botocore) - - yield storage.store(BytesIO(b'test file')) - - conn = storage.connect_s3(*storage.connect_s3.call_args) - bucket = conn.get_bucket(*conn.get_bucket.call_args) - key = bucket.new_key(*bucket.new_key.call_args) - self.assertNotIn( - dict(policy='custom-acl'), - key.set_contents_from_file.call_args - ) - - @defer.inlineCallbacks - def test_store_not_botocore_with_acl(self): - storage = S3FeedStorage( - 's3://mybucket/export.csv', - 'access_key', - 'secret_key', - 'custom-acl' - ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') - self.assertEqual(storage.acl, 'custom-acl') - - storage.is_botocore = False - storage.connect_s3 = mock.MagicMock() - self.assertFalse(storage.is_botocore) - - yield storage.store(BytesIO(b'test file')) - - conn = storage.connect_s3(*storage.connect_s3.call_args) - bucket = conn.get_bucket(*conn.get_bucket.call_args) - key = bucket.new_key(*bucket.new_key.call_args) - self.assertIn( - dict(policy='custom-acl'), - key.set_contents_from_file.call_args - ) - def test_overwrite_default(self): with LogCapture() as log: S3FeedStorage( diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 1dd7031fe..d5b0bb3d8 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -22,7 +22,6 @@ from scrapy.pipelines.files import ( S3FilesStore, ) from scrapy.settings import Settings -from scrapy.utils.boto import is_botocore from scrapy.utils.test import ( assert_aws_environ, assert_gcs_environ, @@ -437,16 +436,10 @@ class TestS3FilesStore(unittest.TestCase): content, key = get_s3_content_and_delete( u.hostname, u.path[1:], with_key=True) self.assertEqual(content, data) - if is_botocore(): - self.assertEqual(key['Metadata'], {'foo': 'bar'}) - self.assertEqual( - key['CacheControl'], S3FilesStore.HEADERS['Cache-Control']) - self.assertEqual(key['ContentType'], 'image/png') - else: - self.assertEqual(key.metadata, {'foo': 'bar'}) - self.assertEqual( - key.cache_control, S3FilesStore.HEADERS['Cache-Control']) - self.assertEqual(key.content_type, 'image/png') + self.assertEqual(key['Metadata'], {'foo': 'bar'}) + self.assertEqual( + key['CacheControl'], S3FilesStore.HEADERS['Cache-Control']) + self.assertEqual(key['ContentType'], 'image/png') class TestGCSFilesStore(unittest.TestCase): From 70c82d33c00538228314dd6cef0253b70f8627e8 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Sun, 20 Sep 2020 16:24:05 +0300 Subject: [PATCH 304/568] httpcompression stats added (#4797) --- scrapy/downloadermiddlewares/httpcompression.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 727c41466..b34f76f21 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -37,6 +37,8 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) + spider.crawler.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + spider.crawler.stats.inc_value('httpcompression/response_count', spider=spider) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) From 7f1e74daa28caa481b4a484392a7dd2a18fc290e Mon Sep 17 00:00:00 2001 From: Mirwaisse Djanbaz Date: Mon, 21 Sep 2020 14:38:16 +0200 Subject: [PATCH 305/568] =?UTF-8?q?dependencides=20=E2=86=92=20dependencie?= =?UTF-8?q?s=20(#4800)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/intro/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 8b4240bf6..fe7bc0a2a 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -211,7 +211,7 @@ PyPy We recommend using the latest PyPy version. The version tested is 5.9.0. For PyPy3, only Linux installation was tested. -Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy. +Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. This means that these dependecies will be built during installation. On macOS, you are likely to face an issue with building Cryptography dependency, solution to this problem is described From 3989f64baa39f7e42b0f798dec15cd250e0fba21 Mon Sep 17 00:00:00 2001 From: Mirwaisse Djanbaz Date: Mon, 21 Sep 2020 14:40:00 +0200 Subject: [PATCH 306/568] =?UTF-8?q?dependecies=20=E2=86=92=20dependencies?= =?UTF-8?q?=20(#4801)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/intro/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index fe7bc0a2a..3bfd3bc3b 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -212,7 +212,7 @@ We recommend using the latest PyPy version. The version tested is 5.9.0. For PyPy3, only Linux installation was tested. Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. -This means that these dependecies will be built during installation. +This means that these dependencies will be built during installation. On macOS, you are likely to face an issue with building Cryptography dependency, solution to this problem is described `here `_, From 008cf1c75ebe72a607149d7efd8f902f8763bc52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 20:45:21 +0200 Subject: [PATCH 307/568] Remove a test that has never been executed in Python 3 --- tests/test_downloader_handlers.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0a374c161..3e8d7e6b9 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -868,29 +868,6 @@ class S3TestCase(unittest.TestCase): self.assertEqual(httpreq.headers['Authorization'], b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') - def test_request_signing5(self): - try: - import botocore # noqa: F401 - except ImportError: - pass - else: - raise unittest.SkipTest( - 'botocore does not support overriding date with x-amz-date') - # deletes an object from the 'johnsmith' bucket using the - # path-style and Date alternative. - date = 'Tue, 27 Mar 2007 21:20:27 +0000' - req = Request( - 's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={ - 'Date': date, - 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000', - }) - with self._mocked_date(date): - httpreq = self.download_request(req, self.spider) - # botocore does not override Date with x-amz-date - self.assertEqual( - httpreq.headers['Authorization'], - b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') - def test_request_signing6(self): # uploads an object to a CNAME style virtual hosted bucket with metadata. date = 'Tue, 27 Mar 2007 21:06:08 +0000' From 56f05fb16476b40f773edc9e93d41c8893a349c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 22:01:09 +0200 Subject: [PATCH 308/568] Use mocking for tests/test_feedexport.py::S3FeedStorageTest::test_store --- tests/test_feedexport.py | 50 +++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 33ac51712..a8c0cf686 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -13,6 +13,7 @@ from logging import getLogger from pathlib import Path from string import ascii_letters, digits from unittest import mock +from unittest.mock import call from urllib.parse import urljoin, urlparse, quote from urllib.request import pathname2url @@ -254,21 +255,42 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): - assert_aws_environ() - uri = os.environ.get('S3_TEST_FILE_URI') - if not uri: - raise unittest.SkipTest("No S3 URI available for testing") - access_key = os.environ.get('AWS_ACCESS_KEY_ID') - secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') - storage = S3FeedStorage(uri, access_key, secret_key) + skip_if_no_boto() + + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + } + crawler = get_crawler(settings_dict=settings) + bucket = 'mybucket' + key = 'export.csv' + storage = S3FeedStorage.from_crawler(crawler, f's3://{bucket}/{key}') verifyObject(IFeedStorage, storage) - file = storage.open(scrapy.Spider("default")) - expected_content = b"content: \xe2\x98\x83" - file.write(expected_content) - yield storage.store(file) - u = urlparse(uri) - content = get_s3_content_and_delete(u.hostname, u.path[1:]) - self.assertEqual(content, expected_content) + + file = mock.MagicMock() + from botocore.stub import Stubber + with Stubber(storage.s3_client) as stub: + stub.add_response( + 'put_object', + expected_params={ + 'Body': file, + 'Bucket': bucket, + 'Key': key, + }, + service_response={}, + ) + + yield storage.store(file) + + stub.assert_no_pending_responses() + self.assertEqual( + file.method_calls, + [ + call.seek(0), + # The call to read does not happen with Stubber + call.close(), + ] + ) def test_init_without_acl(self): storage = S3FeedStorage( From 17e135377a564152e82cd2ad177a6a2749c15832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 22:54:39 +0200 Subject: [PATCH 309/568] Use mocking for tests/test_feedexport.py::BatchDeliveriesTest::test_s3_export --- tests/test_feedexport.py | 99 +++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 51 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index a8c0cf686..ac0b3ccaa 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -13,7 +13,6 @@ from logging import getLogger from pathlib import Path from string import ascii_letters, digits from unittest import mock -from unittest.mock import call from urllib.parse import urljoin, urlparse, quote from urllib.request import pathname2url @@ -42,7 +41,6 @@ from scrapy.extensions.feedexport import ( from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import ( - assert_aws_environ, get_s3_content_and_delete, get_crawler, mock_google_cloud_storage, @@ -286,9 +284,9 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual( file.method_calls, [ - call.seek(0), + mock.call.seek(0), # The call to read does not happen with Stubber - call.close(), + mock.call.close(), ] ) @@ -1518,46 +1516,53 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_s3_export(self): - """ - Test export of items into s3 bucket. - S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini - to perform this test: - [testenv] - setenv = - AWS_SECRET_ACCESS_KEY = ABCD - AWS_ACCESS_KEY_ID = EFGH - S3_TEST_BUCKET_NAME = IJKL - """ - try: - import boto3 - except ImportError: - raise unittest.SkipTest("S3FeedStorage requires boto3") + skip_if_no_boto() - assert_aws_environ() - s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME') - access_key = os.environ.get('AWS_ACCESS_KEY_ID') - secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') - if not s3_test_bucket_name: - raise unittest.SkipTest("No S3 BUCKET available for testing") - - chars = [random.choice(ascii_letters + digits) for _ in range(15)] - filename = ''.join(chars) - prefix = f'tmp/{filename}' - s3_test_file_uri = f's3://{s3_test_bucket_name}/{prefix}/%(batch_time)s.json' - storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) - settings = Settings({ - 'FEEDS': { - s3_test_file_uri: { - 'format': 'json', - }, - }, - 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, - }) + bucket = 'mybucket' items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), ] + + class CustomS3FeedStorage(S3FeedStorage): + + stubs = [] + + def open(self, *args, **kwargs): + from botocore.stub import ANY, Stubber + stub = Stubber(self.s3_client) + stub.activate() + CustomS3FeedStorage.stubs.append(stub) + stub.add_response( + 'put_object', + expected_params={ + 'Body': ANY, + 'Bucket': bucket, + 'Key': ANY, + }, + service_response={}, + ) + return super().open(*args, **kwargs) + + key = 'export.csv' + uri = f's3://{bucket}/{key}/%(batch_time)s.json' + batch_item_count = 1 + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'FEED_EXPORT_BATCH_ITEM_COUNT': batch_item_count, + 'FEED_STORAGES': { + 's3': CustomS3FeedStorage, + }, + 'FEEDS': { + uri: { + 'format': 'json', + }, + }, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, uri) verifyObject(IFeedStorage, storage) class TestSpider(scrapy.Spider): @@ -1567,22 +1572,14 @@ class BatchDeliveriesTest(FeedExportTestBase): for item in items: yield item - s3 = boto3.resource('s3') - my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') - - with MockServer() as s: + with MockServer() as server: runner = CrawlerRunner(Settings(settings)) - TestSpider.start_urls = [s.url('/')] + TestSpider.start_urls = [server.url('/')] yield runner.crawl(TestSpider) - for file_uri in my_bucket.objects.filter(Prefix=prefix): - content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key) - if not content and not items: - break - content = json.loads(content.decode('utf-8')) - expected_batch, items = items[:batch_size], items[batch_size:] - self.assertEqual(expected_batch, content) + self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)+1) + for stub in CustomS3FeedStorage.stubs[:-1]: + stub.assert_no_pending_responses() class FeedExportInitTest(unittest.TestCase): From 35726da434a9bc61dcb1ad5ef475c7d030023c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 22:55:25 +0200 Subject: [PATCH 310/568] tests/test_feedexport.py: remove unused import --- tests/test_feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ac0b3ccaa..18f7f8458 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -13,7 +13,7 @@ from logging import getLogger from pathlib import Path from string import ascii_letters, digits from unittest import mock -from urllib.parse import urljoin, urlparse, quote +from urllib.parse import urljoin, quote from urllib.request import pathname2url import lxml.etree From c3b740f07814107e643bb0d073ea116049e37cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 23:25:37 +0200 Subject: [PATCH 311/568] Use mocking for tests/test_pipeline_files.py::TestS3FilesStore::test_persist --- scrapy/utils/test.py | 9 ---- tests/test_pipeline_files.py | 100 +++++++++++++++++++++++++++-------- 2 files changed, 78 insertions(+), 31 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 94d0ae2d3..cf251442f 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -13,15 +13,6 @@ from twisted.trial.unittest import SkipTest from scrapy.utils.boto import is_botocore_available -def assert_aws_environ(): - """Asserts the current environment is suitable for running AWS testsi. - Raises SkipTest with the reason if it's not. - """ - skip_if_no_boto() - if 'AWS_ACCESS_KEY_ID' not in os.environ: - raise SkipTest("AWS keys not found") - - def assert_gcs_environ(): if 'GCS_PROJECT_ID' not in os.environ: raise SkipTest("GCS_PROJECT_ID not found") diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index d5b0bb3d8..1b14e3b1e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,6 +1,7 @@ import os import random import time +from datetime import datetime from io import BytesIO from shutil import rmtree from tempfile import mkdtemp @@ -23,11 +24,11 @@ from scrapy.pipelines.files import ( ) from scrapy.settings import Settings from scrapy.utils.test import ( - assert_aws_environ, assert_gcs_environ, get_ftp_content_and_delete, get_gcs_content_and_delete, get_s3_content_and_delete, + skip_if_no_boto, ) @@ -414,32 +415,87 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): class TestS3FilesStore(unittest.TestCase): + @defer.inlineCallbacks def test_persist(self): - assert_aws_environ() - uri = os.environ.get('S3_TEST_FILE_URI') - if not uri: - raise unittest.SkipTest("No S3 URI available for testing") - data = b"TestS3FilesStore: \xe2\x98\x83" - buf = BytesIO(data) + skip_if_no_boto() + + bucket = 'mybucket' + key = 'export.csv' + uri = f's3://{bucket}/{key}' + buffer = mock.MagicMock() meta = {'foo': 'bar'} path = '' + content_type = 'image/png' + store = S3FilesStore(uri) - yield store.persist_file( - path, buf, info=None, meta=meta, - headers={'Content-Type': 'image/png'}) - s = yield store.stat_file(path, info=None) - self.assertIn('last_modified', s) - self.assertIn('checksum', s) - self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8') - u = urlparse(uri) - content, key = get_s3_content_and_delete( - u.hostname, u.path[1:], with_key=True) - self.assertEqual(content, data) - self.assertEqual(key['Metadata'], {'foo': 'bar'}) - self.assertEqual( - key['CacheControl'], S3FilesStore.HEADERS['Cache-Control']) - self.assertEqual(key['ContentType'], 'image/png') + from botocore.stub import Stubber + with Stubber(store.s3_client) as stub: + stub.add_response( + 'put_object', + expected_params={ + 'ACL': S3FilesStore.POLICY, + 'Body': buffer, + 'Bucket': bucket, + 'CacheControl': S3FilesStore.HEADERS['Cache-Control'], + 'ContentType': content_type, + 'Key': key, + 'Metadata': meta, + }, + service_response={}, + ) + + yield store.persist_file( + path, + buffer, + info=None, + meta=meta, + headers={'Content-Type': content_type}, + ) + + stub.assert_no_pending_responses() + self.assertEqual( + buffer.method_calls, + [ + mock.call.seek(0), + # The call to read does not happen with Stubber + ] + ) + + @defer.inlineCallbacks + def test_stat(self): + skip_if_no_boto() + + bucket = 'mybucket' + key = 'export.csv' + uri = f's3://{bucket}/{key}' + checksum = '3187896a9657a28163abb31667df64c8' + + store = S3FilesStore(uri) + from botocore.stub import Stubber + with Stubber(store.s3_client) as stub: + stub.add_response( + 'head_object', + expected_params={ + 'Bucket': bucket, + 'Key': key, + }, + service_response={ + 'ETag': f'"{checksum}"', + 'LastModified': datetime(2019, 12, 1), + }, + ) + + file_stats = yield store.stat_file('', info=None) + self.assertEqual( + file_stats, + { + 'checksum': checksum, + 'last_modified': 1575154800, + }, + ) + + stub.assert_no_pending_responses() class TestGCSFilesStore(unittest.TestCase): From 8f46e845190b7affb7a02c6842b465f0c5c3b76c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 23:28:16 +0200 Subject: [PATCH 312/568] Fix style issues --- scrapy/utils/test.py | 12 ------------ tests/test_feedexport.py | 3 +-- tests/test_pipeline_files.py | 1 - 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index cf251442f..24c38283a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -23,18 +23,6 @@ def skip_if_no_boto(): raise SkipTest('missing botocore library') -def get_s3_content_and_delete(bucket, path, with_key=False): - """ Get content from s3 key, and delete key afterwards. - """ - import botocore.session - session = botocore.session.get_session() - client = session.create_client('s3') - key = client.get_object(Bucket=bucket, Key=path) - content = key['Body'].read() - client.delete_object(Bucket=bucket, Key=path) - return (content, key) if with_key else content - - def get_gcs_content_and_delete(bucket, path): from google.cloud import storage client = storage.Client(project=os.environ.get('GCS_PROJECT_ID')) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 18f7f8458..3ed35bec5 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -41,7 +41,6 @@ from scrapy.extensions.feedexport import ( from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import ( - get_s3_content_and_delete, get_crawler, mock_google_cloud_storage, skip_if_no_boto, @@ -1577,7 +1576,7 @@ class BatchDeliveriesTest(FeedExportTestBase): TestSpider.start_urls = [server.url('/')] yield runner.crawl(TestSpider) - self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)+1) + self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1) for stub in CustomS3FeedStorage.stubs[:-1]: stub.assert_no_pending_responses() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 1b14e3b1e..e840298d9 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -27,7 +27,6 @@ from scrapy.utils.test import ( assert_gcs_environ, get_ftp_content_and_delete, get_gcs_content_and_delete, - get_s3_content_and_delete, skip_if_no_boto, ) From 07c1d9c25b6f499fbe2cc0f0139d78cf6c1d204a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 23:32:55 +0200 Subject: [PATCH 313/568] Remove boto3 as a dependency for tests --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 12e40295c..ea356c56a 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,6 @@ deps = -ctests/constraints.txt -rtests/requirements-py3.txt # Extras - boto3>=1.13.0 botocore>=1.4.87 Pillow>=4.0.0 passenv = From c22e810658b227095ea516ed61e51e6be41068ce Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Tue, 22 Sep 2020 07:47:37 +0300 Subject: [PATCH 314/568] httocompression tests added --- .../downloadermiddlewares/httpcompression.py | 9 ++++--- ...st_downloadermiddleware_httpcompression.py | 27 +++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index b34f76f21..ca80e9444 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -18,11 +18,14 @@ except ImportError: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" + def __init__(self, stats): + self.stats = stats + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): raise NotConfigured - return cls() + return cls(crawler.stats) def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', @@ -37,8 +40,8 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) - spider.crawler.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) - spider.crawler.stats.inc_value('httpcompression/response_count', spider=spider) + self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + self.stats.inc_value('httpcompression/response_count', spider=spider) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a806f55ce..998749c2e 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -8,6 +8,7 @@ from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip +from scrapy.utils.test import get_crawler from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -26,8 +27,10 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - self.spider = Spider('foo') - self.mw = HttpCompressionMiddleware() + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('scrapytest.org') + self.mw = HttpCompressionMiddleware(self.crawler.stats) + self.crawler.stats.open_spider(self.spider) def _getresponse(self, coding): if coding not in FORMAT: @@ -50,6 +53,13 @@ class HttpCompressionTest(TestCase): response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'}) return response + def assertStatsEqual(self, key, value): + self.assertEqual( + self.crawler.stats.get_value(key, spider=self.spider), + value, + str(self.crawler.stats.get_stats(self.spider)) + ) + def test_process_request(self): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers @@ -66,6 +76,7 @@ class HttpCompressionTest(TestCase): assert newresponse is not response assert newresponse.body.startswith(b' Date: Tue, 22 Sep 2020 12:45:21 +0200 Subject: [PATCH 315/568] Fix timezone test issue --- tests/test_pipeline_files.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e840298d9..4e1b90787 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -469,6 +469,7 @@ class TestS3FilesStore(unittest.TestCase): key = 'export.csv' uri = f's3://{bucket}/{key}' checksum = '3187896a9657a28163abb31667df64c8' + last_modified = datetime(2019, 12, 1) store = S3FilesStore(uri) from botocore.stub import Stubber @@ -481,7 +482,7 @@ class TestS3FilesStore(unittest.TestCase): }, service_response={ 'ETag': f'"{checksum}"', - 'LastModified': datetime(2019, 12, 1), + 'LastModified': last_modified, }, ) @@ -490,7 +491,7 @@ class TestS3FilesStore(unittest.TestCase): file_stats, { 'checksum': checksum, - 'last_modified': 1575154800, + 'last_modified': last_modified.timestamp(), }, ) From eff96038c7a488860c1c58c2c7e37f888264c3dc Mon Sep 17 00:00:00 2001 From: madeny <7504281+madeny@users.noreply.github.com> Date: Sat, 26 Sep 2020 22:50:38 +0200 Subject: [PATCH 316/568] Correct some typos This won't be an issue if **your** spider doesn't rely on cookies. --- docs/topics/jobs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 58601824a..d855d0133 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -65,7 +65,7 @@ Cookies expiration ------------------ Cookies may expire. So, if you don't resume your spider quickly the requests -scheduled may no longer work. This won't be an issue if you spider doesn't rely +scheduled may no longer work. This won't be an issue if your spider doesn't rely on cookies. From 9186e5a686e04703baf78974824917cf7a5121ec Mon Sep 17 00:00:00 2001 From: dswij Date: Tue, 29 Sep 2020 00:27:39 +0700 Subject: [PATCH 317/568] add pip 20.2 test for tox --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 12e40295c..f73eaef36 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,8 @@ passenv = GCS_PROJECT_ID commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} +install_command = + pip install --use-feature=2020-resolver {opts} {packages} [testenv:typing] basepython = python3 From 894b509d7a4262cfc7548d78e9ff4831c6551c34 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 29 Sep 2020 23:37:28 -0300 Subject: [PATCH 318/568] Crawl rule: remove deprecated code Remove the compatibility layer that handles 'process_request' methods that do not receive a 'response' parameter --- scrapy/spiders/crawl.py | 34 ++++++++++--------------------- tests/test_spider.py | 45 +++++++++++++++++------------------------ 2 files changed, 30 insertions(+), 49 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index bc4551a54..1dcf2e6ab 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,14 +6,11 @@ See documentation in docs/topics/spiders.rst """ import copy -import warnings from typing import Sequence -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider -from scrapy.utils.python import get_func_args from scrapy.utils.spider import iterate_spider_output @@ -37,15 +34,22 @@ _default_link_extractor = LinkExtractor() class Rule: - def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, - process_links=None, process_request=None, errback=None): + def __init__( + self, + link_extractor=None, + callback=None, + cb_kwargs=None, + follow=None, + process_links=None, + process_request=None, + errback=None, + ): self.link_extractor = link_extractor or _default_link_extractor self.callback = callback self.errback = errback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links or _identity self.process_request = process_request or _identity_process_request - self.process_request_argcount = None self.follow = follow if follow is not None else not callback def _compile(self, spider): @@ -53,22 +57,6 @@ class Rule: self.errback = _get_method(self.errback, spider) self.process_links = _get_method(self.process_links, spider) self.process_request = _get_method(self.process_request, spider) - self.process_request_argcount = len(get_func_args(self.process_request)) - if self.process_request_argcount == 1: - warnings.warn( - "Rule.process_request should accept two arguments " - "(request, response), accepting only one is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - - def _process_request(self, request, response): - """ - Wrapper around the request processing function to maintain backward - compatibility with functions that do not take a Response object - """ - args = [request] if self.process_request_argcount == 1 else [request, response] - return self.process_request(*args) class CrawlSpider(Spider): @@ -111,7 +99,7 @@ class CrawlSpider(Spider): for link in rule.process_links(links): seen.add(link) request = self._build_request(rule_index, link) - yield rule._process_request(request, response) + yield rule.process_request(request, response) def _callback(self, response): rule = self._rules[response.meta['rule']] diff --git a/tests/test_spider.py b/tests/test_spider.py index 78157a9b9..d23543f6a 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,8 +1,8 @@ import gzip import inspect -from unittest import mock import warnings from io import BytesIO +from unittest import mock from testfixtures import LogCapture from twisted.trial import unittest @@ -20,7 +20,6 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.linkextractors import LinkExtractor -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.test import get_crawler @@ -280,7 +279,7 @@ class CrawlSpiderTest(SpiderTest): response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) - def process_request_change_domain(request): + def process_request_change_domain(request, response): return request.replace(url=request.url.replace('.org', '.com')) class _CrawlSpider(self.spider_class): @@ -290,17 +289,14 @@ class CrawlSpiderTest(SpiderTest): Rule(LinkExtractor(), process_request=process_request_change_domain), ) - with warnings.catch_warnings(record=True) as cw: - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://example.com/somepage/item/12.html', - 'http://example.com/about.html', - 'http://example.com/nofollow.html']) - self.assertEqual(len(cw), 1) - self.assertEqual(cw[0].category, ScrapyDeprecationWarning) + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://example.com/somepage/item/12.html', + 'http://example.com/about.html', + 'http://example.com/nofollow.html']) def test_process_request_with_response(self): @@ -339,20 +335,17 @@ class CrawlSpiderTest(SpiderTest): Rule(LinkExtractor(), process_request='process_request_upper'), ) - def process_request_upper(self, request): + def process_request_upper(self, request, response): return request.replace(url=request.url.upper()) - with warnings.catch_warnings(record=True) as cw: - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) - self.assertEqual(len(cw), 1) - self.assertEqual(cw[0].category, ScrapyDeprecationWarning) + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', + 'http://EXAMPLE.ORG/ABOUT.HTML', + 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) def test_process_request_instance_method_with_response(self): From 774ebe8796d204b38b332ee71a6ffa678c9b32b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 30 Sep 2020 14:17:30 +0200 Subject: [PATCH 319/568] Mention contributors in the README --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index a8f2ba52b..19faa9a87 100644 --- a/README.rst +++ b/README.rst @@ -34,9 +34,16 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. +Scrapy has contributions from `many users`_ (thanks everyone!) and is sponsored +by `Scrapinghub Ltd`_. + +.. _many users: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapinghub Ltd: https://www.scrapinghub.com/ + Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. + Requirements ============ From 9661a8dcfc79249e8e3e117eb70682ebc8e57a92 Mon Sep 17 00:00:00 2001 From: Sashreek Shankar <45600974+sashreek1@users.noreply.github.com> Date: Thu, 1 Oct 2020 06:46:12 +0530 Subject: [PATCH 320/568] removed datatype specification for *args & ** kwargs --- scrapy/crawler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 4c6b0e496..578016536 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -180,9 +180,9 @@ class CrawlerRunner: :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance, :class:`~scrapy.spiders.Spider` subclass or string - :param list args: arguments to initialize the spider + :param args: arguments to initialize the spider - :param dict kwargs: keyword arguments to initialize the spider + :param kwargs: keyword arguments to initialize the spider """ if isinstance(crawler_or_spidercls, Spider): raise ValueError( From f4629fe2cc8469e540b68a48a00fdcbb5ecd976a Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 14:58:14 +0700 Subject: [PATCH 321/568] Update travis-pip and tox deps conflict for pip20.2 --- .travis.yml | 1 + tox.ini | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b883c5b78..de5bbe9ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,6 +49,7 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi + - python -m pip install --upgrade pip #force travis to use newest version of pip - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index f73eaef36..9a41fc240 100644 --- a/tox.ini +++ b/tox.ini @@ -64,7 +64,8 @@ commands = [pinned] deps = -ctests/constraints.txt - cryptography==2.0 + #using cryptography-2.1.4 for test jobs to solve dependencies conflict on pip20.2>= + cryptography==2.1.4 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 From 66201737a0d245a2568f9b17541d38d6134fa03c Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 15:47:19 +0700 Subject: [PATCH 322/568] fix travis and deps --- .travis.yml | 2 +- tox.ini | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index de5bbe9ab..d53695e01 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,8 +48,8 @@ install: tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + python -m pip install -U pip fi - - python -m pip install --upgrade pip #force travis to use newest version of pip - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index 9a41fc240..abbc0752e 100644 --- a/tox.ini +++ b/tox.ini @@ -64,14 +64,15 @@ commands = [pinned] deps = -ctests/constraints.txt - #using cryptography-2.1.4 for test jobs to solve dependencies conflict on pip20.2>= + #using cryptography-2.1.4 to solve dependencies conflict on pip20.2>= cryptography==2.1.4 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 PyDispatcher==2.0.5 - pyOpenSSL==16.2.0 + #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>= + pyOpenSSL==18.0.0 queuelib==1.4.2 service_identity==16.0.0 Twisted==17.9.0 From 0ea6ff11360963b7a26f11087cb8f62113070dd5 Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 16:14:36 +0700 Subject: [PATCH 323/568] travis pip version and deps --- .travis.yml | 2 +- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d53695e01..1ad5c4e9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,8 +48,8 @@ install: tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - python -m pip install -U pip fi + - pip install --upgrade pip - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index abbc0752e..e923aeb85 100644 --- a/tox.ini +++ b/tox.ini @@ -64,8 +64,8 @@ commands = [pinned] deps = -ctests/constraints.txt - #using cryptography-2.1.4 to solve dependencies conflict on pip20.2>= - cryptography==2.1.4 + #using cryptography-3.1 to solve dependencies conflict on pip20.2>= + cryptography==3.1 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 From f7201b1427749cc364c6f76100f01938d581340c Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 19:59:23 +0700 Subject: [PATCH 324/568] travis and deps --- .travis.yml | 3 ++- tox.ini | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1ad5c4e9b..4c4bc0ef7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,8 @@ matrix: - env: TOXENV=asyncio python: 3.8 dist: bionic +before_install: + - pip install -U pip install: - | if [[ ! -z "$PYPY_VERSION" ]]; then @@ -49,7 +51,6 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - - pip install --upgrade pip - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index e923aeb85..e1e46921e 100644 --- a/tox.ini +++ b/tox.ini @@ -64,8 +64,8 @@ commands = [pinned] deps = -ctests/constraints.txt - #using cryptography-3.1 to solve dependencies conflict on pip20.2>= - cryptography==3.1 + #using cryptography-2.3.1 to solve dependencies conflict on pip20.2>= + cryptography==2.3.1 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 From 392b489a65b62244fa6cc1e1406c8e2e2d50d966 Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 20:32:49 +0700 Subject: [PATCH 325/568] travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4c4bc0ef7..da37c85c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,7 @@ matrix: dist: bionic before_install: - pip install -U pip + - pip --version install: - | if [[ ! -z "$PYPY_VERSION" ]]; then From c83a16898f35c99b0383736e24293edbeaa5d46f Mon Sep 17 00:00:00 2001 From: dswij Date: Thu, 1 Oct 2020 20:59:05 +0700 Subject: [PATCH 326/568] try removing cache in travis to install pip --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index da37c85c6..65fa3fa45 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,9 +63,6 @@ notifications: skip_join: true channels: - irc.freenode.org#scrapy -cache: - directories: - - $HOME/.cache/pip deploy: provider: pypi distributions: "sdist bdist_wheel" From 744f352d09a9ec519c84ec5243e62eef78c66f08 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 1 Oct 2020 14:52:23 -0300 Subject: [PATCH 327/568] Do not process cookies from headers --- scrapy/downloadermiddlewares/cookies.py | 41 ++++++---------------- tests/test_downloadermiddleware_cookies.py | 5 +++ 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 87f8152a4..d95ed3d38 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -97,35 +97,14 @@ class CookiesMiddleware: def _get_request_cookies(self, jar, request): """ - Extract cookies from a Request. Values from the `Request.cookies` attribute - take precedence over values from the `Cookie` request header. + Extract cookies from the Request.cookies attribute """ - def get_cookies_from_header(jar, request): - cookie_header = request.headers.get("Cookie") - if not cookie_header: - return [] - cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";")) - cookie_list_unicode = [] - for cookie_bytes in cookie_gen_bytes: - try: - cookie_unicode = cookie_bytes.decode("utf8") - except UnicodeDecodeError: - logger.warning("Non UTF-8 encoded cookie found in request %s: %s", - request, cookie_bytes) - cookie_unicode = cookie_bytes.decode("latin1", errors="replace") - cookie_list_unicode.append(cookie_unicode) - response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode}) - return jar.make_cookies(response, request) - - def get_cookies_from_attribute(jar, request): - if not request.cookies: - return [] - elif isinstance(request.cookies, dict): - cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) - else: - cookies = request.cookies - formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) - response = Response(request.url, headers={"Set-Cookie": formatted}) - return jar.make_cookies(response, request) - - return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request) + if not request.cookies: + return [] + elif isinstance(request.cookies, dict): + cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) + else: + cookies = request.cookies + formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) + response = Response(request.url, headers={"Set-Cookie": formatted}) + return jar.make_cookies(response, request) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index a3de307ee..aff8542e9 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -2,6 +2,8 @@ import logging from testfixtures import LogCapture from unittest import TestCase +import pytest + from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.exceptions import NotConfigured @@ -243,6 +245,7 @@ class CookiesMiddlewareTest(TestCase): self.assertIn('Cookie', request.headers) self.assertEqual(b'currencyCookie=USD', request.headers['Cookie']) + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty') mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) @@ -257,6 +260,7 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty') + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_header(self): # keep only cookies from 'Cookie' request header req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}) @@ -291,6 +295,7 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req3, self.spider) is None self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')}) From cc81f9ed06399bd9c4e730a76f54032d9a7e9106 Mon Sep 17 00:00:00 2001 From: dswij Date: Fri, 2 Oct 2020 00:56:59 +0700 Subject: [PATCH 328/568] add download setting for tox --- .travis.yml | 4 +--- tox.ini | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 65fa3fa45..5351abf87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,9 +40,6 @@ matrix: - env: TOXENV=asyncio python: 3.8 dist: bionic -before_install: - - pip install -U pip - - pip --version install: - | if [[ ! -z "$PYPY_VERSION" ]]; then @@ -63,6 +60,7 @@ notifications: skip_join: true channels: - irc.freenode.org#scrapy + deploy: provider: pypi distributions: "sdist bdist_wheel" diff --git a/tox.ini b/tox.ini index e1e46921e..f51be432d 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,7 @@ passenv = AWS_SECRET_ACCESS_KEY GCS_TEST_FILE_URI GCS_PROJECT_ID +download = true #allow tox virtualenv to upgarde pip/wheel/setuptools commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = From 95b2e94496f86d1a225d573adc2af2facce35401 Mon Sep 17 00:00:00 2001 From: dswij Date: Fri, 2 Oct 2020 01:05:45 +0700 Subject: [PATCH 329/568] fix comment error on tox and re-add cache for travis --- .travis.yml | 4 +++- tox.ini | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5351abf87..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,9 @@ notifications: skip_join: true channels: - irc.freenode.org#scrapy - +cache: + directories: + - $HOME/.cache/pip deploy: provider: pypi distributions: "sdist bdist_wheel" diff --git a/tox.ini b/tox.ini index f51be432d..8cc522378 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,8 @@ passenv = AWS_SECRET_ACCESS_KEY GCS_TEST_FILE_URI GCS_PROJECT_ID -download = true #allow tox virtualenv to upgarde pip/wheel/setuptools +#allow tox virtualenv to upgrade pip/wheel/setuptools +download = true commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = From 44f0fde9057256dc16f24f22cfebc61626a94450 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 1 Oct 2020 23:21:09 +0500 Subject: [PATCH 330/568] Simplify TLS logging for the modern pyOpenSSL. (#4822) --- scrapy/core/downloader/tls.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index d9f3750d5..b5c6cc895 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -55,18 +55,11 @@ class ScrapyClientTLSOptions(ClientTLSOptions): set_tlsext_host_name(connection, self._hostnameBytes) elif where & SSL.SSL_CB_HANDSHAKE_DONE: if self.verbose_logging: - if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15 - if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0 - logger.debug('SSL connection to %s using protocol %s, cipher %s', - self._hostnameASCII, - connection.get_protocol_version_name(), - connection.get_cipher_name(), - ) - else: - logger.debug('SSL connection to %s using cipher %s', - self._hostnameASCII, - connection.get_cipher_name(), - ) + logger.debug('SSL connection to %s using protocol %s, cipher %s', + self._hostnameASCII, + connection.get_protocol_version_name(), + connection.get_cipher_name(), + ) server_cert = connection.get_peer_certificate() logger.debug('SSL connection certificate: issuer "%s", subject "%s"', x509name_to_string(server_cert.get_issuer()), From e42d82526a1a519ff1305b189acde064a40fbd8d Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 1 Oct 2020 23:22:17 +0500 Subject: [PATCH 331/568] Drop the conditional code for old Twisted (#4820) --- scrapy/core/downloader/tls.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index b5c6cc895..2b8990b75 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -5,7 +5,6 @@ from service_identity.exceptions import CertificateError from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError from twisted.internet.ssl import AcceptableCiphers -from scrapy import twisted_version from scrapy.utils.ssl import x509name_to_string, get_temp_key_info @@ -28,13 +27,6 @@ openssl_methods = { } -if twisted_version < (17, 0, 0): - from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name -else: - def set_tlsext_host_name(connection, hostNameBytes): - connection.set_tlsext_host_name(hostNameBytes) - - class ScrapyClientTLSOptions(ClientTLSOptions): """ SSL Client connection creator ignoring certificate verification errors @@ -52,7 +44,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions): def _identityVerifyingInfoCallback(self, connection, where, ret): if where & SSL.SSL_CB_HANDSHAKE_START: - set_tlsext_host_name(connection, self._hostnameBytes) + connection.set_tlsext_host_name(self._hostnameBytes) elif where & SSL.SSL_CB_HANDSHAKE_DONE: if self.verbose_logging: logger.debug('SSL connection to %s using protocol %s, cipher %s', From f47b120e2b67be7821a06ff77786c0bab2cfece8 Mon Sep 17 00:00:00 2001 From: Habeeb Shopeju Date: Thu, 1 Oct 2020 19:50:11 +0100 Subject: [PATCH 332/568] Documentation of link extractor usage (#4775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added description when using link extractor outside crawlspiders and created reference documentation for scrapy.link.Link class * Added link.rst to toctree * Corrected spelling errors, moved docs to Link doctstring to use autoclass * Moved link docs to link_extractors * Update docs/topics/link-extractors.rst Co-authored-by: Adrián Chaves * Update link.py Improvements to URL description * Update link.py * Update link.py Fixed line length Flake issue * Update link.py Fixed trailing whitespace Co-authored-by: Adrián Chaves --- docs/index.rst | 1 - docs/topics/link-extractors.rst | 21 ++++++++++++++++++--- scrapy/link.py | 17 ++++++++++++++++- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 11aa5c9be..da264fb34 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -78,7 +78,6 @@ Basic concepts topics/settings topics/exceptions - :doc:`topics/commands` Learn about the command-line tool used to manage your Scrapy project. diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index ed32411b0..e12ad45e0 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -10,12 +10,19 @@ The ``__init__`` method of :class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links ` returns a -list of matching :class:`scrapy.link.Link` objects from a +list of matching :class:`~scrapy.link.Link` objects from a :class:`~scrapy.http.Response` object. Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders -through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link -extractors in regular spiders. +through a set of :class:`~scrapy.spiders.Rule` objects. + +You can also use link extractors in regular spiders. For example, you can instantiate +:class:`LinkExtractor ` into a class +variable in your spider, and use it from your spider callbacks:: + + def parse(self, response): + for link in self.link_extractor.extract_links(response): + yield Request(link.url, callback=self.parse) .. _topics-link-extractors-ref: @@ -145,4 +152,12 @@ LxmlLinkExtractor .. automethod:: extract_links +Link +---- + +.. module:: scrapy.link + :synopsis: Link from link extractors + +.. autoclass:: Link + .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py diff --git a/scrapy/link.py b/scrapy/link.py index 684735f6e..e70667361 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -7,7 +7,22 @@ its documentation in: docs/topics/link-extractors.rst class Link: - """Link objects represent an extracted link by the LinkExtractor.""" + """Link objects represent an extracted link by the LinkExtractor. + + Using the anchor tag sample below to illustrate the parameters:: + + Dont follow this one + + :param url: the absolute url being linked to in the anchor tag. + From the sample, this is ``https://example.com/nofollow.html``. + + :param text: the text in the anchor tag. From the sample, this is ``Dont follow this one``. + + :param fragment: the part of the url after the hash symbol. From the sample, this is ``foo``. + + :param nofollow: an indication of the presence or absence of a nofollow value in the ``rel`` attribute + of the anchor tag. + """ __slots__ = ['url', 'text', 'fragment', 'nofollow'] From 159e2b2e2fbbcbb2b083a79e83e37cf4e60ee5ee Mon Sep 17 00:00:00 2001 From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com> Date: Fri, 2 Oct 2020 00:23:08 +0530 Subject: [PATCH 333/568] allowing to run .pyw files (#4646) * allow .pyw in scrapy/commands/runspider.py * aesthetics * added tests for '.pyw' * created class for testing .pyw files * name=None parameter in get_log * small fix * .pyw tests for non-windows * used @skipIf for tests * two more tests skipped --- scrapy/commands/runspider.py | 2 +- tests/test_commands.py | 91 ++++++++++++++++++++++++++++-------- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index aedd8c2ce..b957c29fb 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -11,7 +11,7 @@ def _import_file(filepath): abspath = os.path.abspath(filepath) dirname, file = os.path.split(abspath) fname, fext = os.path.splitext(file) - if fext != '.py': + if fext not in ('.py', '.pyw'): raise ValueError(f"Not a Python source file: {abspath}") if dirname: sys.path = [dirname] + sys.path diff --git a/tests/test_commands.py b/tests/test_commands.py index 2899e5f24..3e54a0948 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -496,6 +496,8 @@ class MiscCommandsTest(CommandTest): class RunSpiderCommandTest(CommandTest): + spider_filename = 'myspider.py' + debug_log_spider = """ import scrapy @@ -507,11 +509,23 @@ class MySpider(scrapy.Spider): return [] """ + badspider = """ +import scrapy + +class BadSpider(scrapy.Spider): + name = "bad" + def start_requests(self): + raise Exception("oops!") + """ + @contextmanager - def _create_file(self, content, name): + def _create_file(self, content, name=None): tmpdir = self.mktemp() os.mkdir(tmpdir) - fname = abspath(join(tmpdir, name)) + if name: + fname = abspath(join(tmpdir, name)) + else: + fname = abspath(join(tmpdir, self.spider_filename)) with open(fname, 'w') as f: f.write(content) try: @@ -519,12 +533,12 @@ class MySpider(scrapy.Spider): finally: rmtree(tmpdir) - def runspider(self, code, name='myspider.py', args=()): + def runspider(self, code, name=None, args=()): with self._create_file(code, name) as fname: return self.proc('runspider', fname, *args) - def get_log(self, code, name='myspider.py', args=()): - p, stdout, stderr = self.runspider(code, name=name, args=args) + def get_log(self, code, name=None, args=()): + p, stdout, stderr = self.runspider(code, name, args=args) return stderr def test_runspider(self): @@ -556,7 +570,7 @@ class MySpider(scrapy.Spider): # which is intended, # but this should not be because of DNS lookup error # assumption: localhost will resolve in all cases (true?) - log = self.get_log(""" + dnscache_spider = """ import scrapy class MySpider(scrapy.Spider): @@ -565,23 +579,20 @@ class MySpider(scrapy.Spider): def parse(self, response): return {'test': 'value'} -""", - args=('-s', 'DNSCACHE_ENABLED=False')) - print(log) +""" + log = self.get_log(dnscache_spider, args=('-s', 'DNSCACHE_ENABLED=False')) self.assertNotIn("DNSLookupError", log) self.assertIn("INFO: Spider opened", log) def test_runspider_log_short_names(self): log1 = self.get_log(self.debug_log_spider, args=('-s', 'LOG_SHORT_NAMES=1')) - print(log1) self.assertIn("[myspider] DEBUG: It Works!", log1) self.assertIn("[scrapy]", log1) self.assertNotIn("[scrapy.core.engine]", log1) log2 = self.get_log(self.debug_log_spider, args=('-s', 'LOG_SHORT_NAMES=0')) - print(log2) self.assertIn("[myspider] DEBUG: It Works!", log2) self.assertNotIn("[scrapy]", log2) self.assertIn("[scrapy.core.engine]", log2) @@ -599,15 +610,7 @@ class MySpider(scrapy.Spider): self.assertIn('Unable to load', log) def test_start_requests_errors(self): - log = self.get_log(""" -import scrapy - -class BadSpider(scrapy.Spider): - name = "bad" - def start_requests(self): - raise Exception("oops!") - """, name="badspider.py") - print(log) + log = self.get_log(self.badspider, name='badspider.py') self.assertIn("start_requests", log) self.assertIn("badspider.py", log) @@ -696,6 +699,54 @@ class MySpider(scrapy.Spider): self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) +class WindowsRunSpiderCommandTest(RunSpiderCommandTest): + + spider_filename = 'myspider.pyw' + + def setUp(self): + super(WindowsRunSpiderCommandTest, self).setUp() + + def test_start_requests_errors(self): + log = self.get_log(self.badspider, name='badspider.pyw') + self.assertIn("start_requests", log) + self.assertIn("badspider.pyw", log) + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_run_good_spider(self): + super().test_run_good_spider() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider(self): + super().test_runspider() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider_dnscache_disabled(self): + super().test_runspider_dnscache_disabled() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider_log_level(self): + super().test_runspider_log_level() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider_log_short_names(self): + super().test_runspider_log_short_names() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider_no_spider_found(self): + super().test_runspider_no_spider_found() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_output(self): + super().test_output() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_overwrite_output(self): + super().test_overwrite_output() + + def test_runspider_unable_to_load(self): + raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") + + class BenchCommandTest(CommandTest): def test_run(self): From 797a6690c07ae90f7a2d703832e2fe44466d656f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 1 Oct 2020 23:11:11 -0300 Subject: [PATCH 334/568] Tests: use classes instead of paths in settings (#4817) --- tests/test_dupefilters.py | 10 +-- tests/test_logformatter.py | 4 +- tests/test_pipelines.py | 2 +- tests/test_request_attribute_binding.py | 14 ++-- tests/test_request_cb_kwargs.py | 4 +- tests/test_spidermiddleware_output_chain.py | 92 ++++++++++----------- 6 files changed, 63 insertions(+), 63 deletions(-) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 95a4fca0d..680bb6dc8 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -43,7 +43,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_crawler_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -51,14 +51,14 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_settings_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'} + 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'from_settings') def test_df_direct_scheduler(self): - settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'} + settings = {'DUPEFILTER_CLASS': DirectDupeFilter} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, 'n/a') @@ -162,7 +162,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(SimpleSpider, settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) @@ -191,7 +191,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(SimpleSpider, settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index dc5be398f..6381f895b 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -193,7 +193,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): self.base_settings = { 'LOG_LEVEL': 'DEBUG', 'ITEM_PIPELINES': { - __name__ + '.DropSomeItemsPipeline': 300, + DropSomeItemsPipeline: 300, }, } @@ -212,7 +212,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): @defer.inlineCallbacks def test_skip_messages(self): settings = self.base_settings.copy() - settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter' + settings['LOG_FORMATTER'] = SkipMessagesLogFormatter crawler = CrawlerRunner(settings).create_crawler(ItemSpider) with LogCapture() as lc: yield crawler.crawl(mockserver=self.mockserver) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index c72f1a338..ff3af9a74 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -68,7 +68,7 @@ class PipelineTestCase(unittest.TestCase): def _create_crawler(self, pipeline_class): settings = { - 'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1}, + 'ITEM_PIPELINES': {pipeline_class: 1}, } crawler = get_crawler(ItemSpider, settings) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 907117468..00c532c41 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -92,7 +92,7 @@ class CrawlTestCase(TestCase): url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".RaiseExceptionRequestMiddleware": 590, + RaiseExceptionRequestMiddleware: 590, }, }) crawler = runner.create_crawler(SingleRequestSpider) @@ -119,7 +119,7 @@ class CrawlTestCase(TestCase): url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".ProcessResponseMiddleware": 595, + ProcessResponseMiddleware: 595, } }) crawler = runner.create_crawler(SingleRequestSpider) @@ -149,8 +149,8 @@ class CrawlTestCase(TestCase): url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".RaiseExceptionRequestMiddleware": 590, - __name__ + ".CatchExceptionOverrideRequestMiddleware": 595, + RaiseExceptionRequestMiddleware: 590, + CatchExceptionOverrideRequestMiddleware: 595, }, }) crawler = runner.create_crawler(SingleRequestSpider) @@ -170,8 +170,8 @@ class CrawlTestCase(TestCase): url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".RaiseExceptionRequestMiddleware": 590, - __name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595, + RaiseExceptionRequestMiddleware: 590, + CatchExceptionDoNotOverrideRequestMiddleware: 595, }, }) crawler = runner.create_crawler(SingleRequestSpider) @@ -188,7 +188,7 @@ class CrawlTestCase(TestCase): """ runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".AlternativeCallbacksMiddleware": 595, + AlternativeCallbacksMiddleware: 595, } }) crawler = runner.create_crawler(AlternativeCallbacksSpider) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index bd49179aa..145a4e9b2 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -50,10 +50,10 @@ class KeywordArgumentsSpider(MockServerSpider): name = 'kwargs' custom_settings = { 'DOWNLOADER_MIDDLEWARES': { - __name__ + '.InjectArgumentsDownloaderMiddleware': 750, + InjectArgumentsDownloaderMiddleware: 750, }, 'SPIDER_MIDDLEWARES': { - __name__ + '.InjectArgumentsSpiderMiddleware': 750, + InjectArgumentsSpiderMiddleware: 750, }, } diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 2f454addc..029bf8bd6 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -16,11 +16,20 @@ class LogExceptionMiddleware: # ================================================================================ # (0) recover from an exception on a spider callback +class RecoveryMiddleware: + def process_spider_exception(self, response, exception, spider): + spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) + return [ + {'from': 'process_spider_exception'}, + Request(response.url, meta={'dont_fail': True}, dont_filter=True), + ] + + class RecoverySpider(Spider): name = 'RecoverySpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.RecoveryMiddleware': 10, + RecoveryMiddleware: 10, }, } @@ -34,15 +43,6 @@ class RecoverySpider(Spider): raise TabError() -class RecoveryMiddleware: - def process_spider_exception(self, response, exception, spider): - spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) - return [ - {'from': 'process_spider_exception'}, - Request(response.url, meta={'dont_fail': True}, dont_filter=True), - ] - - # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method class FailProcessSpiderInputMiddleware: @@ -56,9 +56,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): custom_settings = { 'SPIDER_MIDDLEWARES': { # spider - __name__ + '.LogExceptionMiddleware': 10, - __name__ + '.FailProcessSpiderInputMiddleware': 8, - __name__ + '.LogExceptionMiddleware': 6, + FailProcessSpiderInputMiddleware: 8, + LogExceptionMiddleware: 6, # engine } } @@ -87,7 +86,7 @@ class GeneratorCallbackSpider(Spider): name = 'GeneratorCallbackSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 10, + LogExceptionMiddleware: 10, }, } @@ -106,7 +105,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider) name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 100000, + LogExceptionMiddleware: 100000, }, } @@ -117,7 +116,7 @@ class NotGeneratorCallbackSpider(Spider): name = 'NotGeneratorCallbackSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 10, + LogExceptionMiddleware: 10, }, } @@ -134,32 +133,13 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS name = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 100000, + LogExceptionMiddleware: 100000, }, } # ================================================================================ # (4) exceptions from a middleware process_spider_output method (generator) -class GeneratorOutputChainSpider(Spider): - name = 'GeneratorOutputChainSpider' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - __name__ + '.GeneratorFailMiddleware': 10, - __name__ + '.GeneratorDoNothingAfterFailureMiddleware': 8, - __name__ + '.GeneratorRecoverMiddleware': 5, - __name__ + '.GeneratorDoNothingAfterRecoveryMiddleware': 3, - }, - } - - def start_requests(self): - yield Request(self.mockserver.url('/status?n=200')) - - def parse(self, response): - yield {'processed': ['parse-first-item']} - yield {'processed': ['parse-second-item']} - - class _GeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): for r in result: @@ -205,26 +185,28 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): pass -# ================================================================================ -# (5) exceptions from a middleware process_spider_output method (not generator) -class NotGeneratorOutputChainSpider(Spider): - name = 'NotGeneratorOutputChainSpider' +class GeneratorOutputChainSpider(Spider): + name = 'GeneratorOutputChainSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.NotGeneratorFailMiddleware': 10, - __name__ + '.NotGeneratorDoNothingAfterFailureMiddleware': 8, - __name__ + '.NotGeneratorRecoverMiddleware': 5, - __name__ + '.NotGeneratorDoNothingAfterRecoveryMiddleware': 3, + GeneratorFailMiddleware: 10, + GeneratorDoNothingAfterFailureMiddleware: 8, + GeneratorRecoverMiddleware: 5, + GeneratorDoNothingAfterRecoveryMiddleware: 3, }, } def start_requests(self): - return [Request(self.mockserver.url('/status?n=200'))] + yield Request(self.mockserver.url('/status?n=200')) def parse(self, response): - return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}] + yield {'processed': ['parse-first-item']} + yield {'processed': ['parse-second-item']} +# ================================================================================ +# (5) exceptions from a middleware process_spider_output method (not generator) + class _NotGeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): out = [] @@ -276,6 +258,24 @@ class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddlew pass +class NotGeneratorOutputChainSpider(Spider): + name = 'NotGeneratorOutputChainSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + NotGeneratorFailMiddleware: 10, + NotGeneratorDoNothingAfterFailureMiddleware: 8, + NotGeneratorRecoverMiddleware: 5, + NotGeneratorDoNothingAfterRecoveryMiddleware: 3, + }, + } + + def start_requests(self): + return [Request(self.mockserver.url('/status?n=200'))] + + def parse(self, response): + return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}] + + # ================================================================================ class TestSpiderMiddleware(TestCase): @classmethod From 004b40a7193a7c0c5138abe764cad6d1d9a4fc57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 3 Oct 2020 00:53:55 +0200 Subject: [PATCH 335/568] =?UTF-8?q?as=20soon=20as=20=E2=86=92=20as=20long?= =?UTF-8?q?=20as=20(#4825)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 675f55c38..4d2580a6c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -140,7 +140,7 @@ original pull request author hasn't had time to address them. In this case consider picking up this pull request: open a new pull request with all commits from the original pull request, as well as additional changes to address the raised issues. Doing so helps a lot; it is -not considered rude as soon as the original author is acknowledged by keeping +not considered rude as long as the original author is acknowledged by keeping his/her commits. You can pull an existing pull request to a local branch From 892dd9da57c5cf005670743c8abfdfffc7027acc Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 21:00:58 +0100 Subject: [PATCH 336/568] Adding support for zstd in HttpCompressionMiddleware --- scrapy/downloadermiddlewares/httpcompression.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 727c41466..b1abb8b5c 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -14,6 +14,12 @@ try: except ImportError: pass +try: + import zstd + ACCEPTED_ENCODINGS.append(b'zstd') +except ImportError: + pass + class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be @@ -67,4 +73,6 @@ class HttpCompressionMiddleware: body = zlib.decompress(body, -15) if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: body = brotli.decompress(body) + if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS: + body = zstd.decompress(body) return body From c6c3f2ce661ef9296302b9d489d5a2cb2e49f481 Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 21:10:40 +0100 Subject: [PATCH 337/568] Updating the doc entry for the HTTP compress downloader middleware on zstd --- docs/topics/downloader-middleware.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 06e614941..9645ed5fd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -684,11 +684,14 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ responses, - provided `brotlipy`_ is installed. + This middleware also supports decoding `brotli-compressed`_ as well as + `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is + installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://pypi.org/project/brotlipy/ +.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt +.. _zstd: https://pypi.org/project/zstd/ HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From da3171d4f71d50a15b08b76b2b1e06bddfa56694 Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 23:18:58 +0100 Subject: [PATCH 338/568] Using the `zstandard` package than `zstd` for supporting frames both with and without the content size info See also: https://github.com/sergey-dryabzhinsky/python-zstd/issues/53 --- docs/topics/downloader-middleware.rst | 4 ++-- scrapy/downloadermiddlewares/httpcompression.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 9645ed5fd..7c63a623d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -685,13 +685,13 @@ HttpCompressionMiddleware sent/received from web sites. This middleware also supports decoding `brotli-compressed`_ as well as - `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is + `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://pypi.org/project/brotlipy/ .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt -.. _zstd: https://pypi.org/project/zstd/ +.. _zstandard: https://pypi.org/project/zstandard/ HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index b1abb8b5c..56421a6ba 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,4 +1,5 @@ import zlib +import io from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse @@ -15,7 +16,7 @@ except ImportError: pass try: - import zstd + import zstandard ACCEPTED_ENCODINGS.append(b'zstd') except ImportError: pass @@ -74,5 +75,8 @@ class HttpCompressionMiddleware: if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: body = brotli.decompress(body) if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS: - body = zstd.decompress(body) + # Using its streaming API since its simple API could handle only cases + # where there is content size data embedded in the frame + reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body)) + body = reader.read() return body From 50e1f35d1fcaeb2e0a0c4fb29894cb7c686a33cd Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 23:43:12 +0100 Subject: [PATCH 339/568] Adding test cases for the zstd content encoding --- tests/requirements-py3.txt | 3 +- .../html-zstd-static-content-size.bin | Bin 0 -> 8066 bytes .../html-zstd-static-no-content-size.bin | Bin 0 -> 8063 bytes .../html-zstd-streaming-no-content-size.bin | Bin 0 -> 8047 bytes ...st_downloadermiddleware_httpcompression.py | 26 ++++++++++++++++++ 5 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/sample_data/compressed/html-zstd-static-content-size.bin create mode 100644 tests/sample_data/compressed/html-zstd-static-no-content-size.bin create mode 100644 tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2247ed917..2eed2f5da 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -16,6 +16,7 @@ uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython -brotlipy +brotlipy # optional for HTTP compress downloader middleware tests +zstandard # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tests/sample_data/compressed/html-zstd-static-content-size.bin b/tests/sample_data/compressed/html-zstd-static-content-size.bin new file mode 100644 index 0000000000000000000000000000000000000000..b5c2038e893481e2835ac74bc790e3718986e426 GIT binary patch literal 8066 zcmV-|AAR5`wJ-goRU`obmHPmitfU$&;EqO3lFJAV16>@^C8(hY1ZsEGeDXVhj51gD z{wbwTMW0fDVIjsC8zUwJLIPd^Tmj940%8FN4lqZfU`(TE6bp>O3tV-7e>YaR^FV=)2Knk9h* zPD2Bxu`I2bEJ}oPC|O!VVVHKB!!*Tl8rtL0DmWw>I4vT=w8gSua2St>L(&)r1jK1r zB93OkEFwfW4Gr3a353%c%i_@@6d(g3YGzZS2 zoTW`Hi$=w=8$L-PHPy)01wDyvS31MEJ`~NhjNz41g9-ZJB>s+5DPeP;2ej=V*+6! z2AsyCEXzc~31Ax}lf`M!Fs)Hq1*RRwGC2*?lnG^m!iYdlH$+BRYv$Pg*Bn!uBmIj$fL>Nb+ zI3N}nq$wB$#lixsfDwsMEIcR{9u$&CQHqNO3cLt#DP1=;z9HvN0E*SB0B*i|V@? zo!0dgG3lazM?J5RBBIe@s@IB`{`D4B*O(MBjS}6Wt}`j>9cd;_G)lU*(nS1E>Agxb zjnrOTx<i{{ zuB!54rdz~ydTv6L_*KKIR4?Lcu5WP{s>|c5E+#Mj_g#pa>{ZT1y12Y}n{th}_Fg9A z1<pTU+@2?hd|hVob(Mx!|A9L6+?2gL$o znujl=Vp?bP`Io;tI|2zX28DP;Bu<1Tv0&gpczD1Z49GyhvY;TT5Mhqv;Q{e@YIOW5 zqWax`XR5Q;t%z4Y--^b(_2;8+t;tDU>hpQ0m`+OFBE|bpS6Zlk`l0deZdK^3BEqA> zRR2pCDs;Ng>RwcLUCNkuG%jD}5BD@tF&)x8I_jk&p7V)G(`gZ{6xZdtwNm_#i_Tl2 zqB?K&qnD;qh`TiXT=d>C?`4E=@0jSv^u2Ns{X||@`%Yx~m|nN3bgg-Nx_tMh zb2;^SUoY!f553Y~t6QVm_vRBZ^WMz=)_7=tdGooqRzH&-Za(+c=|tQ_UuU{hqXP@< z0cji$4~m6_Csd@4D6L2Ibahm`g!G*LBSHufrT@5&`E=gt9~CLC;}Jc7L|=%~6FWqh z)KQ{@tS;?4N|cF5C{yuIl!%f;qC}+9QL>bGMwkhqyiClabV{lyoz%Zjs#H$uh%&8( zGE+h*FQsBA^U|pZX{r@dh?4q3dQWF3rYIBo$(bpW`jX0&^eGck=d==2g+9fQD#cPN zrs*jaN-9yMJ4&e7Vo6!qdOH>KdRm#P*!oJDnd^{Z>vwA1tum=Ey;6m#)M8huSXwL< zyYE+pnzz`xLRIF)^nSQ}mAF#nVoIqle<}8*>gtp&UCI#3`$D>wuEmgpUg~u*WtaXF zV@FG=YEsvw?77xb#?qIJe2AxK>MKiD`*%9?=$NK5X1y|B^~%4xGPYPE{XHF$5%uVv zh$RPg#J-Nu4u_8E88LN0hKV6`NVMXQA;#D(2W5Cv4reS8VvI4C3^^pk7z44y7-P{K z3Za>!Q7F?WBrt{yiq@z^(6q(i%pd>(0YCr&0Mioy02mAmh>Sx5DrwgN6o3I56b29! zEDRI~0t7&C;P4u`C=>t&LSZl%C>~Ta1x^BJ0QDms#I{8t+xDvw^)H%OmM^UtD6u=U zJi2g;L;Z>qIeYC%x77AvCUM0~2dZsvqsQWjcw9QI^3oQ^-Ny=pBn5wXbuZyGOE-C_ zU+(88f@qO{Z!@13Ld9B{bEg_PV51mS_}EE@qw|Xtd&KjUfOP~dg4w^u)KKYBv)CAw zot38EujidM6)6_Q0emIE5v2Dd^J_S#jAd|478Ri)!r?XqW)yq}yGjGyFZpbke@%9A z_^u{F`#3YR{645{L-~!AtK$yGb6vk!62X2+S8;$e6dz#?0lkbchOLSGA;3MZNDi7_ zDZ`<`7%Pq~Ov@c1`J>Q+#XEwZ=En{ju0tq)SgwM`F;(vui(^FjAxj@&3Tt+_cON8; zz{D;>Q?B)}jJ^@EIH)ri__uIYjQ$Y@q47ge6H$0uQ(bqJX;zw;fYKs74qQlWnM$zn zt42$TaFlT17RDKQ`q8l82lUt`FP*I7d0CAn3p5-cre}g%ko2+aBjS01-TT!%Qrs`2 zL~I4ZXS$tk!Q*FY&%tSqr}IP3GN6x2d0K&-ZUNiTB0Ivy7Z|@j7m;$}>sJ?+$a1=&4s!@7FS6X}iQx6KvgDT?Bz{7oilbL9z-4ct_rAsPzI!Y^_3E}j__m*@@3lVL2ujan z-u(=F9dJ)WDyvlfO1+zK;~Mmg=xq%#`vU6msw@-M!y$M-#>{el>f_pp|83onyRevw z@EiId#6snzKtlyy;h30OWXlx=z~ z8}Cu#3gc># zIsI;^khN5tfp}$v>?3qU-7kD0p8a@sT=af|0#_@H>GQR&&9K?&p^xx*zgWVa9Vz4P z4;Zh}^^L+z-surWhn#H`tOz}3p7HvZ9Kw{D2R?l=7)&}Gyqjjo#btC!7Syv>l%_4S zLwFzbs!lxJBv+9Xhv{~*+wa~&-|>*CZ`3;@nD5bjTf>=$P3rs}kA#>Wz7~BbuZJv8 zndrY)4{tW7znl}>9`ep3Kh#XT?jiGkI#$4%dBoD`P5%KpD0O2=r;|Mb<&a&A2?8>sh%T{-p_2KWE-H%9O1BV(V`{m17l>qV~z=?BrOh;_ok7@rd2|~ z`IqZ~j#yB*?T5UQcoG{c>rM7YkW2>8;YPtzq;4M3q*%5%@@zBzElOkE`!xUm@V@_h zF-3O)mbFwQ7h|5XRm0Ma9d?>`gzZKP8)duDY^I3gof79a|Lels+-KtuK4VlhC3Ih0 zS@X39#s>(w_aSJpT9w=&J}k{$ads~zVPbbbhdHgGLH>|G%lp-{0F#l;doyZWe3rMJ*W&^`p7joRO!v022)e(S+p z#yehuj3s;f^${>C5j_r4M@v^RvCZ_?fQoR~@4}WtcGlRGBi|pxdi$syZ}exD$01+G z!M>kq1o{kDtgmOKHpPtot=pS5O4zuF=()lO#Q8LVERrYO{T8<%5qO!rCdcTd9Diy& zC!{vY%$8?enEN)b92Nl>-FXKbAROmvSiFp3vPTh4Zp%8Z=gHJ?K>4LnS(xX zb1J2eqR#`H`qoU?o?WSpX&zX!Zl6h?>1f9m)UeGc;iMW6RBN` zkl5>1k(4{f2WZS$^!@ zVpweBLB^Ep_DMvN#Cg|2ESpjZP&ZnkE5Kw{5h08s;?v_4zyDJ|M$~D7c0cAczQA*4 z7-2EajlyM1j2A#K9ukMp75cR6ap^JDiQXLOfGnP;iKT?t%&rN zW6;7Lzin2>bV9bWHd_5g$q_v)YP1k_D3ry?S7nHbVHw+=Vu%rQXzPTumK-5KS(A;& zI_u;DQK;-3XAyWu3o+}Y=g>xm!&@972xxn$eg8vt_ardqgyqTVVZNKWmp~shY`27o zq69i#6~P3nHkYnM3*&Hgg-N^nKa0yH`ZIH~mG|}O!Cn<_vP~wR2{kxG{GrA*RvY8K zNliJ&ZTjR9^!=>yVcRHcZv-zA@5v#vp zYI;Fe6~Qj#_X>8S-WlaX9v?%^mbl-kD|H4URprK77}T9wsfA$5#tsiB@w$k-t2hKB z;ckC-UZ+?VQq@0oR}SiGfG>2g(5`^|$T}f|n$P~Ac@h%RoBOO)QL0U-2!P%wips{d zsLED_O>Q@N_{5ZgmBp$Ed)<`1hO#8uDj?_B5&nnuv)yVNfeq``N6E(e$>H=yi3F5z zzy_?9I!OZLvNVpiytoK=mlJ$ulD1hq-oO!53_>++*WX4Vi1}7lS2-&}5bnJ|t~%?P zMTEb0rtS#zI2RWRHDwG}h3{y=&8>Qr>j)!3u*(hn3>26YL%s@o1my&q359knG`dqz zyk30)IO4i98PPUCphScEb9w~$piA#FaT3dUWv)o-pKBElLM2yeoAZMUpHI8uN>O^v zQO}LjuKd*pH2f1}na}!3u<@8A)Sw#kF4XQ89j;t)w(`+N4DWmn?d z2+i9@u~Dq`ZcPfB=IjZG?h=BI+}}+Lkz+7!EQf@RHL1(nSRA_-oF9*2U@q;UN@u$VVQ8KO9Bvr|F6nB#ZEtf8?B^8gnu^v z`WG&m zKEQJZfNxH;;~fY175Z-j&m#Hdqmg+R-GHQ&gD9o0HrWqwurD#R-ySmGv-=HI^c1gD zwYgO}Sti&zRVeh`-zlbaJxPTod6hmnHB5!=sR#GjjqI&I4d)OoY{jCiG%^9l`AFErDd9JHjyE;%@t zoSwtaPOn#wVgJ$`LE{=$fGi zEM+B+x6g!hLK!By3B{`z+qL$?*CYWGQ)=tpGL@Ll5^qV$>QEAfhw)z*nPM$S6o7Ih zPB}WUZ@V^N5TKb_H){!DoiWxV@iZfO7$#tj3H9|I10G4Lhw9o@v5pZbX^G0yBDE>E z|M1D4{8|9ZO#=&)kYbd&S{&&EXvJ17;={#0B+H$nB)%W7rnMddqR5PS43SNe;E;F9 zE+QMlb_G*0toS}$mRuj?y+v7epym)z7Z*RiJ!kDUyGx!0q#goF z1lU@}erQBm5)WJ5R6+dPE02?nDoLSL6FPT->dG~d1yVNUjUUQs69|>6qQVy4P6z8egty%z#uULZ&s2jW?Fp4iQKq^Wz~HyVK$F)P4V7DLN8Y9!+F`+s znMjga|7GOl_+X&4Yx4(;8>S(-{XLklu7{62fB4o9#sA7TODSq|iYeuGFw-i$ALJwN z?YCcejmWGw_gsJF8AQ>ZjFHOfvhE@DzBc6g;L7?@tUm03-g@0*e(pV7BHJZZakVL5 z3tSnbG5*j7S#h&Gcvoe0R%iolUJz&v@cCRHj!~e_TnU>%gmH^RUWYh7Vn#D;m~?tb zuU91Uf>ZwMkbrR1IxN3DxJybgk`Q>`iCW05&3=^63x1eOF14G3enY>+)Go5O#604MLGWJRHEUjnhpxgKg>cKb(@=Fvn|J%$(wPu(&hPAF+r8GnOx ztum~If0)stw2OxFuNI4bk5~l2m}^0?mriM`sW;69uY9h=Vb(Cf0*U2sy4{>uAw%B zmEQ2}+14&Cu-OiA%TFJYs#r*A z)qynN(DPUDsJJ9+gOoC+Aw|wc5S6{q50eQ*p^r`s1kvFnwbU|(kkUi$93rM%zaD9d zKM;gFhj z^O4=x(ZU?v!;*H1)LPX`2>=bGQEkTmJFYLS`d8S(-k}egdju$|jTk*FF*uV6U_@|@ zF67`#tjJ{MOP91@OW2{?20f)Y;;NuYLPh#Mog7UaG>(aKx2S@$}hMGhrsiA+qxmf%?3EFI|SAoj}a zXG$P0BuNp$3bkPn{;4;MdRR7kMEh=P=&L&u9D0*M-p~(25WEBby}_n^<(CAU57?P`b1-VsB6%%MEM$O4iO*V8pw;o%2mG0`0C^h>T2oiL&; z4R&bCN1-EepFzb5&1BdoF~aR!R0X6}B043_OI3o-I+&2q(r7uR4r0D>!=QmuFB_dc z#Cd;!C_GL`*pmI&4>q!5tu`kwjfe|zTvjZtTG1)=g;$!8TSusve#Lmrp?xxO6q0;2 ztc4FD_D7#paqN}5hyVE8I#)3PpWNT+=kcq#uruz8Pg&StbtUaE1Dv%t&A)%2qlZS?rpybhE#F9kknmP+7Fn%e(p)Bikzjs zcR?+z>z0)1fs*;ryQ|0mNP;L{`(hO2G*75pX;+M>D%f45d<|;iFVkU)W7BDdCoHDZMKHGZ}fDahIn9VA30Ww&x=UM=SKb$wa>@yLXOClaN6gNTHP z9DQv@Q1xLW;JQ&`4&{bmhJ4BgRg(**gUYT=D?zte@jkjMMa3q3vpvVv^M8SRBpUl2 zLpemb6)#?PfMRYM6i*kHj+U?%`$pV0?7{x{_$IY_oK@u zkRPiRQarhDiz4%x$WRVtGT*2`$(^|mp!j-<>F~%-hSAHL!e7lv79~ymqYD$oGd}~T z=R=M&7xyD#K)|>|OF%;A~C=c3?=8w2fzC4ql1WI<>&7| zi6DkQ?l1Z<9A<3{Ihn03Q1PPFrcD%B04SP1R7AY5;9mXqSsB}?!;SRcQB;W1l1+49 zCkOOzJ=c@$Gv`Yyw%v9W&-~bM6XTwaveGIflSPj82XY4+q$E4hO4k>PuzU$I5*T=* z0xBms9%iY1P9R$;^u2JzEj2V#T(%A=Tq*u^&kdP^-hMCmgHpsS%<+F6QqbD_d<^+2 zq&&gVmRy3$wWQod1t5diDF+o(>iFdl+UU@Vu{EE6gwipxWqEJS-4zp>)t+s z+CmgEpIf%5cVC5QEM7S;&F zpNMBaX`fV?!T{488eyRFVNiT%K0z2oY|(O+v~@Z_7nW}3Nx zmCsSsxh)?|OR~R|-qkyEAi>n?Lp$x2wsrgVwa~IJ2yx zg)qj0`!t6(&At=G4uAmQ0mNC-E+nFiNfqr0baw3p?4DvAIAwv&#u3OmDi(ND^by8T zEVKXBZi%IJWhNr%lMy*J+w zbV-M7DA?|fpWc_gx{Y5?yTCy_hgBu0ex(2j`f+xKbP1SEm7n*Pa1r{74DruGNsbVL zTw49`*DnkEFstbn-5>JUWa{LCm0L@QeZ1cOrfa?3T5F%eR~bs5QGO*Wckg~uFtwW| z`Cm6sdO>HzPn}}t7n9^DRpfn%JJn|rX!jD&(yLv~rmxBWEZ6n(3m>aN+TM0m-*t+9}@VeBC4QW82|tP literal 0 HcmV?d00001 diff --git a/tests/sample_data/compressed/html-zstd-static-no-content-size.bin b/tests/sample_data/compressed/html-zstd-static-no-content-size.bin new file mode 100644 index 0000000000000000000000000000000000000000..3d494192e2c72294810309eb14e8d55e0431fd1c GIT binary patch literal 8063 zcmV-_AAsN}wJ-eyIF~>I5=%-jHWmwButy&FfHONt+7lRiP9buC_okxG);TpGzz4dXqx1-M#8ky6bI)x zG!o@Bi=#bSV_BRAaV(Yzlf{Gykw(I_$)bVN9t(&9Iv#{dt=WU^pFYb;7T5QlP>$ONY?N;{22IS>mtaNrz=#bW|tA_kns zqAbfq!U z0Z>pT8WEhvqA(&7OFPZdnk6D}9F0=pFj~_n5XrJ&nzOVPaU=`JX_f|=NJJP%qBtNH z7^Eo}1;xSwtAG)SP%Jzs79JFmM^l!CgCl{&LV@FuSS)90k4KAGK(O{`ltyvR0%I(i zL$Safkj9}{U=SieAaEWq00Be+3pfBk5lC?}eXVd+jp*mfb+R!RpI3#OcZ=$~8lBel z6*1|ee@8v9ks_keVXD`PnEv$^Ro9plF^v-4qOLP3>K$n&O*Bfnw$eoWPwBl%GmX?< zT)IZaU7G1Ty}P|Ds&mn)dJ(@)Uf&P>t9*~i20*$Q%YQ4C$9CUiHr&rS8s1Z2=Nw=>mM!q3-u5#y01^w{Vn2J7d_#fl&-4s zVy0Wfb$V_>l=xNET+FY!#f5A2gvd#ca?g9Y{^6=zlrGb)N_)|ZU*|e^U00zla?+po zbn%`BmUF!3Brp)MJ`>)Lxrpm$UOgrLN2{wm zO=#WYU42p4$HQmpdVOX7w|RFf)0e0B+f>}kREczbr|ZAXg_vHqsdTM*d%Ar0rgJ&< zd0#K;Y*U z4-blkg(p;`jwr21^mKJpyoB_e{v$#N5vBjQj`?)n=^qs-uHz9se?(u1(i1yGnAB0C zgsd*@J4%#^M<`SAPn3v~L!v~a(owRMcSe{Ap}b7YqjXBDD4o>5P^wf;>WDI}g)&n@ zC@-aADD%>(2x+PnQ;3rKLV8bUC#EP9`pKCollqd%l=LYRQs=Z1Q-waokSfJeDyHcv z6-p{mr8`Qf*kVap*?Kz_^LkpDs@VEUnVIX5V(WKm-K{dIFTGNQsnlXusaRSp6}#_O zg_^h6x5n_xnmJB&0#25py#291I915YC zqfsc+C?qh342ssMM9{Rw;LIQZ0Rcb&0RYnz000;a42X20~#l7$_c8GzCrqXaMyi9mKXpA=~z=5%n*cSe7rX87Q$kvpl+R zi$nd26FGbBNw?JYU?y?JOb4oMZ==WJiFjN(t@6?q$KA&YgCqricy%w~G)p&ms9)~q zCxU2^e{VCN7DB~ZnRBNaIbfq0RruIRhoke06nn(;lz?>vErQv<#?(;hQM1?>m7SHQ z-mmAKHWeur#Q}UJz!9YPB=c)Hr;KHAOcoWPBEsP|1ZEU`2fIoG-7ooUn14-narmw# zLHjr}v;01&ZA1Bul&j+o$8%l3SQ5d0Nmp@zG!!3U4FSE3Fovy({2{*kTT@+km1$O*n1IqEJPuq)ZJA22@vBBl zig1*0;1?7iNf!+JnJW|{*qeN^4 z!e_djZo%VcYR|!GqdrGaSYM@6!tPi5=8+jQ0?X=S!<6n8>IG%IfoSVP)+iIyE}0OV zL#OjY&N85nN_kp=oNfWz(IPv-#upgBJ{OU4;_F=dwc&mburUsT9QqbC6?cSFjTzyv z%CB#Y|A}=Rs%mBg{+PlsvEy#xZkh2XM$evI462Lh$eyse1hw{~e0&T4wBd3kckv%C zS1zz9`v1S-@u}ZK{&Cu8R}Knq^WMH6f#C}R8wG)g%2(+@%U}(KxTyUy+*Z=FS4lk= zb?**ykLamaRPWa^VQIU>Q4?(4T3rNzZWo~)OT8Y7EjOc9*G5Z4Q@v1Bn*25uqKgP^*e(K}eiT`cgkh`#$itroy zAjCrDr9eXkUg4Oqx;okXoBE|%iMbMM{Xlr=qn1jw@3eagE5kwb^M54o3%Ll*<#YtN zQMdjR!bA{fw^~iqGWMq={2^f4#lrPLVzY^`^JKv9L2ZT+WsEZ%B7kGCc)wV}o*gOU?hhER z(e;hOOy21cMu(hj6s!n6W}fl-mmI>BnFl_7G8jxc9K4%m$i-!JNfy+zSCpnLvqN|v z^r}ug-6U6$6o=_{vfJ<8Lf`R_sc+OfBbe{eeOtqshfV7I9*=~W9=;ZRD6fYsPnqby zR}XJCr@x#N+aB`HBR|wkyzU|Me>zscnt8<1=}rFuIw*BxNvD%N0_BihiwOxw>&*HM z(|6#X19f8)w5gsWklxR1qhuSP%N*gbaM7YA>jPtCzGIFFrX(#6llP{P$EH<6!11Pl(WF?mIPz>W|1C;m-TO5E|M0&5doe|K z0hYB?Bo||zvQ@*nx%VMxv09bfAU-V0h+DrSXw@`38f+0-wfmhLvJu{9Rtu&PwHMsiQX(k{YV279 zGpilv6LfwEfi`e0J?vc}8KF?I7RAL9KfC&T*fjHh~*Yi&y_`gmbPb|{tW6rmU=6Bgs$z#H;T6iR2At_ zOt1@hhWMG?uW6nNn>@z$NBw&Ai0HY(2*mj`fh>|I-2E1}AQ5<(ye7x!r5t~1JSU_! z%FLE$U6}heuN)Qu7~OdX93UL$YFNCCVX{XNPHxLOuII_-Ij!pcDDe??PpG?5vGr}a z%6Q#2oPJ2($TE!))I$4I?lq@zMSP}6m1APoLPoUThI>>n6L|`=G3`4QESZBoaC0i9 zkEDBnbw<10xh*3#HD!-bL%TL((x-A7w=~G~UHL{yAqT%ZDMfkK;kXs1qB!tvQn_B4 zr2?47!_e{p$TPm30^E)#6F1mK9qv?GexK_G)^N&SX%D_MT*-C|g5 z<3Yxh>-I@RlEit}K`fh62~am$p)0^-RuLhLBI48I6uv8EZ)rsC5=zuJ)V)vC%bTO(_5x!GNZymYP+*p{5sW>zK zVlCJxp~bz*xO4QZmCBM}3W3}ik~XGxW=O```o=|_z#wTcC8_#{8?#HF?xr&SvEk80 zTVo0$=TBE>qP$>PFp7f|V|OvNA4*?w3ZwpKv|QG$2#le z0#T^!9A^=DM+-6Qr038^hQnJNAqZ%DseS)LcK0MO=Y-|S>S4Z{xtBm6G;FtoiJ}BL zUKPOvt2UReL<{3^b%ja0`#+1zCHgaSvX%Gs>A_wVZ?a7$p9wWMMEs$~HC7wrzDZ3v z$8GxL5%m46@nPF2Yi|T!uSBzCIb}}pczx1KX;rC?S?=o!U}p*cv+Mfs$`Px-VQP9o zR~5l7|sVj8`B30$aS{T%wTB(I#%Ek^4C-J(7ysJ0_BjIj; zcV4Gh7E;webyp7RYJe|vu+Xl6{Kz^XgPPC&pm`D!(wqCNRZ*%y@s+R+A1LD*b)AR^|Re-8-We$)kn$3`pMz+Mu`NJaKHwv zl{!fRX6SB39r!Og9DlEHt`PP`qA! z0XX8iGa1n~K%hi}`g3{&_@GPgGjS5jd1bCh>Yr;B4?-nZX`Azd3!hKB;!06^%~8*d z)2{s02Q>T>WSP(UNwD#lB-Ef9^DflxS7f`H$92v5wUj76}mTWU}44aeCJg;)(?w-VvzbVf_cHc;TP0= z-DQuwZO-H0s9IT>2WvXLvBL-?wfjGwyZQRg#v2|vD=l6I*Mec(=YjEjD^G(WXWJXsY%N?mRv+Rz@1)8wM*y$+$(~W3^WqRmT>E?Wo@H0!+z8Fv zMzK+>^=?fHn&#{Yi0%@Cj@;i(3z1_mZY+m{jWwyu+gKdC7n~oDVPG!p%&FlqN0zlX zTE0O7qCQzpgKsz&nxK!~>gsl)iO5GDgg+cb?d3k`DF3g@JH<{vfE%r(MTCDg{`wa# zn&gn)C{ho9aaLfgiir1%4sl})YeO<~m)>Ah3D!ajhY7EUD0Wg}+7cYS4>a4n-T1}TKErjhGuA99;^K<_PeTRz@H2cmY`irh{ik-&%4x)V(1 z55C>CaRsXJkdvIK#fx%bnjOguFEg5?=THCmgHWsW&Y!+|3*Y1 zKs<1y2K_jFm@Kd(XDS&!mvbSSWgfwICDeJd`HXm_mh%b?R4+8snjEyG#x6NHmzx(UUr7~8e>!`CDM6H{vI-ZGV#&Ju4)%IZ)OhKKQA7nx!$NECo_Bu+Ux zv2VLJU=X00S~qJ6VVyD7B=IyOc^D>OjtTYk9RnUos)y>@Rk4l{DQSty(;~GgxBu|T zp8Q$>%S{6dlaOMRx>_9R18BuoE#kw)ha?~U`@y`F0G(MneLdzXhm`;)|8pkT0BgaTFy+AO$r zc|>z;ZOXA6HfQ{FKzXfD?}K%i8aao}#w5#~qa?l`ucoyg0;0%_c?^+FlHibc$}S=s z!*&HzF|7DLT$l;$S7m6}l!Nl;-H8uNeo~!HMX@7Xhczr;^(6^xc2{RCG?s^_9n6yJ zrx$FhmFiv|kG(}%cA(}EP!|_JzCCB{Hsu&U?6}s1J$$f1IRGJo{JTq@1*9GVN(9(i z#(ro-S`rUi-BdyR+bfTgjVei@RTDaQg6hgOkp)sV<&7W8X%h&Qs-nUc-Q`CQQ8HD` z{^C$fc1{QDJcPI1BgPcLG0#+kBkc*5N>Qe|7{K7S#XytS7!8$MY)9Uv9NJ;QjhRT2 zTK{F_ay-3^u9La`ryj?QLH}ffZlrDV}9;ETq4^gRdKZ`Ukh9r zq%r=`23c{lJa|`SbyjEtZC(&)4eyUtO)H*D`Jh)3rF_I8?-;0;u&*kv(G3bz~Vx01q&(qS)Y5(p>+xsJ08CjMw zj96&m3T@9*Isi5zobu@pzkOFpkTK}tChPK9PY`x-`%jdq)M69fH}ByLdR$Yu9@Q=h z+u?sUMS+N?L;xr6qhv*)ZC?Vh%DEnDUUvIOljhMxRXv6n2~XWJAxd3TLBshLF-j?i?bfT)!S^ia!v9 zb$N?J+4zS_k3~knp*_PTrzkI@xN2Fgdk6yrUP{tZv@y)$V<9LDydiD|pc|yB!t;^c z*U`cp-NTZ0iPT!vO9=oCq)~0g|2wWPt@>Bk!rq||ntKE&s*M;uEHOBf31CEUjV|Qi zORUIb=1Z5fU`ya_?aZ}1#Abkf&@814#@Mn1!Tsf&W**r(9XhI9JvgO5bf1?|kKISBp67iH8wnXi zGBziPrB~%RSJ}N_K zE+k12!3wou5dNt*i+Wf#dPMtfYUryw6C8SzLEg|0LlDA~+&gK6$U@fuv1B(i6WP#! zbu0RQS@)pN+9kI=?ColY%ia+~eaxXez{moU5ZBW+S>fRaWiinl?DR{n6P+-kEe&>P z%15Ciai2lO3C(2KCo#h9TvP?5RU$ei%}Z5+&N`To(9&o*rVe7hal@d2QZE~wKE!!{ zfG9jpNZ6A7*bg?cVy!kOFO7%`aa>j`tyF4pQxv(?ticeYCV09(!Faw@v$ngXdK$~E}R~O#- z=sZ(Z}-Z^5+F4B2@(Yve207!x;Ui)Gca%cnTUB*QwczeKQ+r#6y7Ld`OIPXfsg3;yw2fcK-zCXgSi z6;eF8Z;K-HnaEHMWisEWK*^oC51{yZis|skPKMFTo5EksNfsqd{G$sK#WOzxr{_bC zGZ*(GVnD#SLrXwIW@(*}E4JNs70>+Ga1-O6jA#V(Ms1Bim-eMF%lSfq5>)> zI38xHeNG@-DfGQ?#4R;6Q(U$VDO@T3bk7Z$g5G{F_=8f!EX?tL9a7NR`+N-fDx^HY z(Ux3-%C)Kp&Y8un&v|F3^NC_#JkvU8u$0fLK4OW;1Qa_W!MMaSxLLSdZ|mMZg4#k9 zGM`(vsCeouYh`#%KB$zT&0vbp;x2;M4MJA(;U*~3np@aJ1-s-{fPUy6K}3pzLe|67 z;}D^W4%}L;ENmB3|F2R`y2p@GU`VDWvV>}7WpHM~ink?_Ga`9u8YPGIa~9SJ#Gi;~ zKWU#-nZf|m92#Mu@?lVXXg)z0T&wb-A0J=?DH(CQ?3n!D#j|%!#}r^Bs)|iw4GZ?q zC4JPB6q*!Of!zHjxUfYtC8C`N(1x2%HdhVy)@;#om9%v_Kp1s)8xK){`evHBfR)ct z)a3pOMMWetiqN0)^p(rHABV!>05>p)B#T)LJSWuVsl5pJ21UKd-$O4-Os3!@(=Z$a zdtysai~}M!jCy7L_ts(qY1wrm((ihE8CMER*1I!vlbb*G8Mmv+vV+#N?>MupqJ=QV zgZngxHqE{h#SVY~-~q&0(k>*Tj7b&k33PVt1?-+;95`ix&BhVPIw}@;RP+(XP%N|! zH(RO|3em0IZ)KNlmMD}mJPF1r+sPXP#1qSi1*lgj*6NX7rpoAmp-G3$BE2`?5Ohh0 zY$(|7j-TF_zPgQHPrJZDJcm^!sD7mY3Hot%hI9#-O_iVbmT(dJiVX43LP?Ggf?Qht z@YgR3`!K8N7Tq86*ktPDf|XlKh<&`?{-$fa-CAp(!dDqepHY4#D|hdHQZTifCi!1C zP&}Os3O%?TsHs! literal 0 HcmV?d00001 diff --git a/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin b/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin new file mode 100644 index 0000000000000000000000000000000000000000..97bdbcae01d8838c3ee046f227b91b99893daabe GIT binary patch literal 8047 zcmV-#ACTZEwJ-eySQYyK`m2ZriV$moT92z_% zjb-wPIGDz0$^#6}5>XnZX%w0S38$gK!b~Pkn^+oiz+qbRU~nFb37FO_2_$eD8ZeDz zY0YF&BAi3X(i#fGw9_1>DUQ?79*>mIZ^uctjkM#yB7#PQwy$Gz(@C zA;M{B&?ZbEoYq(tj~1Z-X%GxfV~Id`s6euKaF_$(;lXJft&tq1MJyte#?l(5J#d!E z#93O?D3YZ$iy~<(%Azb7h%=cut?_V3NSKHN6O5%T7F0M5jb*}ElnACda2Dk(ZDLt8 z62-x3OJg*}At7Pf1czx6XK9UP(nyr{m_Pxtkf3SW1E*0S%|z2Ar!^9$ou)WA$DxrZ zr&%2B(HhI*G>BudOqeVtOo%iRrcD+Noc35i6bcrQWpSFqIH#cv&`=%|k_7`iAd|=f zrm-}KM3Zn@!#D_RW%p$t8Q`OT0J3h(xcq-Uao(*Di@{8bgR-{^y1gK&Ry44sEeHR=RIBgruQ~; zeYfz7e5(-Ei@2KWTik`}^0=yt$&3Ge7vd&+m2;6UE-&7uT;r|1m&tem^s6*^jf=)r zxpec^*1h~DU#>#9ZyprqaLE{$)+|kefqgQY0n796X^sO~Hoj#v;is_`(EmFMybftyrrym;c?pB4qDk3~8RR2pCDs;Ng>RwcL zUCNkuG%jD}5BD@tF&)x8I_f3jIiHv`ofgqbab2!kE5-k~=)4sws`FMqdTAX|w3+w@D91jnQg@q$h zM@5v@BYL_zDqcc*PX7@hgox6AT*rJm@AQv~6xZ>Ho+Mv`>uF`GV(TkqX0Ahut>39rt4!)kuT)_wwb)fEmR4pSyYE+pnzz`xLgvNv zez<&Xa>A$`H!?Lb{f&#gKzu>UA+?m;MuDM@y+{QrD&Ixzd`$BOAhLYeI2144jt1o zV(Nel6GP~bXvH5xjImn|%J8Th&R8PE7-K9Ma!80V24aaZ#-ceCLNiCBP^M8xU<{c6 zW6+A!s8sN@wcyMk0099&0098g6951h3=D`&Ljo#o*Z~xP0U8tr5ELv76bJ$YKycs) z6R{{100u%~Fc2snR5S%n0%!pBBOPGdQ;==LD-nJdO=c4|$1C}PGP^V5qHDG|4zGZh zGuoa6mpUIw5*KB$2&)LTzs6=^=~3KbQ!DjV8nIu?cXB|a zEQ>?>3gr>1_ax;Pa}Z4|B4ke5@^yrrZ7847!w!y0bF*K2HhgkThvLXtO-T1~R3KwN zbg&IoG}3s-AjgxqUydGO{3V6r7}qdy1gMGYM(7M15&82CbX-9nG({<+pM%j~9B-KB zA7SrN;DeP%1k~n7H*D=gSo*N+gJzzpi!at8g7+ae9)X{_cQ_w^IFG>6FS=Gv|FD4H z2%82T2t z8JQJ_s^P=o*v64g>*6U`ja}~PhuZ|aQkJ+aFY76dfyNx#&Wm+|0$f9}2I-MkP)~oys1m zZxWugw@wlJtRZz@ARO<|WdeFQF7GFjSvE?2eLK0|rX6zg7Jd;6hdxcQ0Db9dsGvI> znO4`eHtVHcs8)$QOD!Lcg+A_5*%xT}p90Hp4FCL1l6P=64=nrA)U%VV z=35zSR1(z?&~mYeUqH09BMhF5`1d@UVW*675)Kx<>wE-4H=PRQ%8Zo55)>|r=qPO5 zZ!q$P)-SW0(EgqO9o&WE6+eK1aw4vN!!X`69@m1ag<|@DLj{_pwuSjCAjl)=ME$l= zghyPx5P4q&`~-2rDnW`;sTL9bf{HxswJU~3eZ3QVJITL<#^G^0yZ76#&}T8^ z@s09-1l9L&Z|gSmL=;87hag+x`<~f9H0dGRQ{KhzyTTjh>EGwXnjQ{&WY`a9YU0Tc z8UE>IJc8OIj7}f*N5MhC8;d+0{s_oJKJj8*I1YG=)0uu%_Z%8D*3_odj*xIaM>ncP zU%mE-(ZVGjEmh6oJF69W3bD2Z&uw!$SjI`v-uAI8q3xbxBmyV{f`q%s5QQ#nm;HbzfAg$!m>{kE}TSu(#Njl|&IAz+i}5 z#3Q)Z6gV2@BLKB~)D2S+UO?8((^$<5Ue!|Al-xG95Ch|{9XSxPeKDUOIbH%8TlR+XBT!c&`VInXOG;y6 z^7LA95V5o0ST4Wstbvpx-5>4u=36@y(Vs1k<15DjydRgRmTTC3e-pRleQMPx;@>dEj8t^N06a? zn6c1PIgT497foJ?Hc6#`-%lxpoV7b{QB$`b@W)}yI0{Tr3t%Z8hL%u3o^kC2OZz_g zxXC=~aHnGW{dB2z7A|2bL15Dzzs#Jc0vxyTWpeFW!Xwb$DB95?X7~4q1?|M)H01u( zAGh>|Yz2zBey`i8T$$UgBvCW2=W{zx^44%_0pNzVqg&IW2oF-xwhrNL-gB^wi6k?uVqM*+)n-^FpCWm#n+ixUor|2$&oZXb z%&?Aiv5kxGf!Vr%O;T(Qw**L@>!t$yvES(0t1%govsYKwKzWhaaGwUv2c22ayt#WB^7zJ+Wu*(e`!v*W)9WQKId zO@&B)?Y ziMqTP~yYlOs^Uvj}Me0h;2%5hSPqtgOM3LCjsLEdgs5W=3#Jh12jbfxd^PeROauGXo z_Li6M>0wfpaa=@9i!3!lQ2v448cPG?9#T`Bf3yDN(S1MY@zHFQuN$F!R~BP#Ws#ir zcv|TdpsLhYQSOBbV2qXJ*@gR{+Y#K~U@<*%Rh8;4AbUl(QKLTOgE>Bq(3V8ssaSPJ z@T$stEmhXt-cy9ZjN%UGCsDh2Vpq8YBi3#Ys$b`^tZS-&S1U(=PpCGNR%ik>3}>1$ zL$?0@a6bvw(<9s4IgpuKK%U&Fj4mK8we z*v9Y&@Yy1?&9@DU>_;PF{rPabjS3wP8o~xS;+jODf*^LAoNY^l`^pK=Y>#cak5};# zc7jmOZ5IGSNdA#;rTQvGBLsWze{+Re&)|px%1KJV$&eTcU$Uq6I>QCzt9D=Sw&$)V# z$(5FWUg+s)@l2JR(A$6=95B9OoZvCNM%Oc}3|AKE!;+B+GM=ycgrRqwKhz*N=Ed9Y zR+QV!IJ&7BVBYem@Q3BU!m4fF8JCoYL%bpu?rOj&t%wOk%lbA*wD%~7Xq3oui12;$ z%H50GU~xrw+_$OH5G#=O1r8$_{uW}Teq>y^d&R-P6`&bBv>{aQA3oOt&2 z;z^m+jzE6%>z-jbXAU8fu=dw9Fi)?9fe}J&8#ANi*1I_=*q-z8Uv!sN9o6}6K_Wi` z;;~#UEytv;{YJsqH8|fW!@zIa_gjPi91&@uxBL$PLPR9VG04GO?u?m4y{=WMCc+5h z0WZVRZLcJQ4*&eBsZ*r$1KbFmXhb+?+*+VH4-|mObq4|op;axBTMqWcr)*@Q1?w_A&GblrN-tsp{*>izh`P0zL#Smnq?lr zc_rj|v*C?6rE0hQbneK3z8N<*@#oV8#Xned#K@uxTEbkwWsULZ%1T>7qwG1 zQc0DOA6j>sVs+PwHIxS#%+U;%1xpxSIPrenvvWOwWB>Nbp0xb@A zi!i~G#eTT%km92cJ{VLbpct0aOyG(Q3fKQqaSfQq$M#6TjO3#Z_R`kpg#=pK+O8yu z*u;8`_Arg{NgO_X-VbI(ls8tTUs~F*qDX|Jb|AzLtE4<|Hs$FEmw*f+oBKopr_9>cMdI6Taqa+Z+|v0Z^QhA+Plnq~q&ptiL( z<*mu_?uSE`pR{^Y#o5utA)V!OwEgxrv8|+XRZGDypy@cDd8TxJ(7Vzkn5!8aLfq4f@WHU`iFe%y#4eGy%U)==bi|zJVO-mkumT| zT=qX~|7)aN9~{LW#czjr=&f=+W~T<1Q6l>#Rk5`xXH`xa%$e&j16i?H9+<0M$RlXO z&_!=87O3Nq9$oS>qhW7F+(-Q2m`%lJHdS7MsG}^=TcwAFmgV$ZM5U05SJ8V61i2C{$@>xd0sE4jHllVZs z;34hodJDxMFH!Y5wpluqaXb5~Fvu~>`%RF}_714cS(RiNe#b_k_#$tTg~H;u8|_2C ztpibDP&eH@(IE)yYD|lRY1wU;&TMvSxaBAhy9q4Bw(2vze0cn^K4>nv_8`R=Y1E#x zA`qoe`9t>uA@ZX;0znfvNr?OzLlb-OZ4S|r>*Gc`><=(&UC#d@y#68XvE+z8+}LoY zDN@KNaZ*d8Jxr&8mr7d7Z;ZP7SeD5G{)!tK6oYhCc!sh2Kv|gEa`;>^a^U<1hhAU?T#DB7>uC0x%1%zCwok5|#0rd`V49lM+~F&3aLCxIzm8t@VN_rx|A z)M9PkAMe_lHgafkZ=O*-Gh}O(RlzecPnB&}+%?8}DB@%K^o+v7G=va%5?6>OAxbd8 z;&e78YyTw!WOLrwp$yCn36X1*`iL-tFk7HZ>;q(buf0sxAIf>D1e+)Y=C*~xu!bBe z3`IZ|1czaTvo-a*EiG=)kH|dV4kAu_EFP_7p2}+wE@YU>*tH~JykQ@P0F1}6cfg3P3*7-C$!_RmvZDd&R92Sv`hDcK15#j#=QFzmQVN(&MADm#th&Oj# z8sR9!VJFzwyJFM#MXogXx9*~thKe>cC)H#sDI^GJSTY|M#g9&?;xjJk9{BIQb+2MN zKDjm1|M5ZP(AH8FowTss>q^_o3~+g0>X9n1cCDjIXxe{yMS z(Bno0WHAuVMI>PJsCb*-y6(5zhH2-MGv))>KI(KXA2E^r#o2+-kB*LF&pBJp%8)Ma zgVqEKES{3(s(Y$8;U=%B!16tl{b?AcO`dybs$%(4PhC)1qPR=?q8YyV(Sa)l{BjWO z>ApBBDa{ivSJ@R4s|tG-310)6hn8s}0oruht1IH5JAB((6a z)JP#89c#tL4yUl(-gmCv#PR{OA+uQsZUoo16|GjH^Wh;3_E(^k+>PCzIFeMEN=9#7bCZR)inDx1xjv-;l|1r};lj9tjTp238If+)5cQH^DPvOxzmP%HI;R zO++NsOj;2q`Hh0B&3>^be@j0)AVqC;1R5=fAfhI@gbFhbl4mr66ye$hoD|TIaIUZl z@a`rf9?03+uLo-6Ig4bFsaezClMv-62LFy00Q~65Nan|`FcjbJ+oDL^Oly`S8O3f? zu*`Pu0w{VdiZOX4t;5LLO{Z4pC3BS~?xUNDBAFk+sevK4lZ!h!F#u@XaZEravw({x zfr`nl1}~J!*<_p$EwLD2D25XA<$+%P|ItH0u>$kgpG1)2AFo$^_!?$R4E@Yj5~>(e zs?#P4Jb)LPu9PCOYr#4F?pYZx)bU1o-cdw}(#a-@yVFGUXFbTOYR-rVBA$a&7B z2ufK$VZ8K4e8C0*V|)f^mT`c=@>OM{AxRA#EWFnXW5YRWwJIu`s+$Xi%Nd zMqrA|<6eqa3_@1;FjEvIlUvxM0#qOLAA{xpb1@!oQX(g6rnR`X->_& zAIFWu;cj3INrtr8e@-YTsYems4T^D*e~VtM8&JVX7Q>(k@Wdsds73c~R(fU6Zy$^O zwq>_SWccvFGR_J1y?1BmCO3ZU8E&_nWmkNo9p#L!ilV|u)_122ttVV36k-6_hlgQ~ z#$A3fGCGZPH{RJV1uWf=@sr6;m>BU-)@Cf&R54^2X*p@*(rjNVC@ifV-IDFnY&DSM zPW6mg-^uF^l&6m22C%n~o7JPi<=NCR!1Khpss=hTD#`$)reM9NkF+URcs3JH;VtoJr literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a806f55ce..63a69f7af 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -20,6 +20,12 @@ FORMAT = { 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), 'br': ('html-br.bin', 'br'), + # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin + 'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'), + # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin + 'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), + # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin + 'zstd-streaming-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), } @@ -80,6 +86,26 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b" Date: Mon, 5 Oct 2020 23:55:48 +0100 Subject: [PATCH 340/568] Minor adjustment to the test case in tests/test_downloadermiddleware_httpcompression.py --- tests/test_downloadermiddleware_httpcompression.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 63a69f7af..7e9f9cc5d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -101,7 +101,8 @@ class HttpCompressionTest(TestCase): newresponse = self.mw.process_response(request, response, self.spider) if raw_content is None: raw_content = newresponse.body - assert raw_content == newresponse.body + else: + assert raw_content == newresponse.body assert newresponse is not response assert newresponse.body.startswith(b" Date: Tue, 6 Oct 2020 19:44:48 +0700 Subject: [PATCH 341/568] Explicitly declare PyDispatcher as dependencies --- .travis.yml | 1 + tox.ini | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b883c5b78..fb4802070 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,7 @@ install: tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + $HOME/virtualenvs/$PYPY_VERSION/bin/pypy -m ensurepip --default-pip fi - pip install -U tox twine wheel codecov diff --git a/tox.ini b/tox.ini index 8cc522378..10b144619 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,7 @@ deps = pyOpenSSL==18.0.0 queuelib==1.4.2 service_identity==16.0.0 - Twisted==17.9.0 + Twisted==20.3.0 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt @@ -115,6 +115,9 @@ deps = {[testenv:pinned]deps} [testenv:pypy3] basepython = pypy3 +deps = + {[testenv]deps} + PyDispatcher==2.0.5 commands = py.test {posargs:--durations=10 docs scrapy tests} @@ -124,7 +127,6 @@ commands = {[testenv:pypy3]commands} deps = {[pinned]deps} lxml==4.0.0 - PyPyDispatcher==2.1.0 [docs] changedir = docs From ce6884d517abb75b884ae23ab59e1405daa83187 Mon Sep 17 00:00:00 2001 From: dswij <44697459+dswij@users.noreply.github.com> Date: Tue, 6 Oct 2020 19:51:42 +0700 Subject: [PATCH 342/568] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 10b144619..4e6ca3ec9 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,7 @@ deps = service_identity==16.0.0 Twisted==20.3.0 w3lib==1.17.0 - zope.interface==4.1.3 + zope.interface==5.1.2 -rtests/requirements-py3.txt # Extras botocore==1.4.87 From 137c8ba6ee393d0887373c88a9f252af4efd8e3c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 6 Oct 2020 10:50:17 -0300 Subject: [PATCH 343/568] Docs: mention limitation about Cookie header --- docs/topics/downloader-middleware.rst | 5 +++++ docs/topics/request-response.rst | 12 ++++++++++++ docs/topics/settings.rst | 5 +++++ 3 files changed, 22 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 06e614941..ae84b54fb 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -207,6 +207,11 @@ CookiesMiddleware a warning. Refer to :ref:`topics-logging-advanced-customization` to customize the logging behaviour. + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + The following settings can be used to configure the cookie middleware: * :setting:`COOKIES_ENABLED` diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 30b1945d0..f3aaa2c8f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -61,6 +61,12 @@ Request objects :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If ``None`` is passed as value, the HTTP header will not be sent at all. + + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + :type headers: dict :param cookies: the request cookies. These can be sent in two forms. @@ -102,6 +108,12 @@ Request objects ) For more info see :ref:`cookies-mw`. + + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + :type cookies: dict or list :param encoding: the encoding of this request (defaults to ``'utf-8'``). diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 06234c5d9..71331c841 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -352,6 +352,11 @@ Default:: The default headers used for Scrapy HTTP Requests. They're populated in the :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`. +.. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + .. setting:: DEPTH_LIMIT DEPTH_LIMIT From 9b1f86b613d2039b0a66ba2b527a7e9bffadaf3a Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 6 Oct 2020 18:50:55 +0500 Subject: [PATCH 344/568] Use f-strings --- scrapy/utils/iterators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index e140e3f6f..3b504e56a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -35,7 +35,7 @@ def xmliter(obj, nodename): namespaces = {} if header_end: for tagname in reversed(re.findall(END_TAG_RE, header_end)): - tag = re.search(r'<\s*%s.*?xmlns[:=][^>]*>' % tagname, text[:header_end_idx[1]], re.S) + tag = re.search(fr'<\s*{tagname}.*?xmlns[:=][^>]*>', text[:header_end_idx[1]], re.S) if tag: namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group())) @@ -45,7 +45,7 @@ def xmliter(obj, nodename): document_header + match.group().replace( nodename, - '%s %s' % (nodename, ' '.join(namespaces.values())), + f'{nodename} {" ".join(namespaces.values())}', 1 ) + header_end @@ -56,7 +56,7 @@ def xmliter(obj, nodename): def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): from lxml import etree reader = _StreamReader(obj) - tag = f'{{{namespace}}}{nodename}'if namespace else nodename + tag = f'{{{namespace}}}{nodename}' if namespace else nodename iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename) for _, node in iterable: From 6050604f626a5ee38239ef1eff44d0c867469a3b Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Tue, 6 Oct 2020 18:59:57 +0300 Subject: [PATCH 345/568] httocompression/response_bytes tests added --- tests/test_downloadermiddleware_httpcompression.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 998749c2e..144fd3378 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -77,6 +77,7 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b' Date: Tue, 6 Oct 2020 19:13:29 +0200 Subject: [PATCH 346/568] Do not consider about: URLs invalid --- scrapy/http/request/__init__.py | 6 +++++- tests/test_http_request.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index ef58deacc..498f1b052 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -65,7 +65,11 @@ class Request(object_ref): s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) - if ('://' not in self._url) and (not self._url.startswith('data:')): + if ( + '://' not in self._url + and not self._url.startswith('about:') + and not self._url.startswith('data:') + ): raise ValueError(f'Missing scheme in request url: {self._url}') url = property(_get_url, obsolete_setter(_set_url, 'url')) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 0a303dbe2..74579dfc4 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -43,6 +43,15 @@ class RequestTest(unittest.TestCase): assert r.headers is not headers self.assertEqual(r.headers[b"caca"], b"coco") + def test_url_scheme(self): + # This test passes by not raising any (ValueError) exception + self.request_class('http://example.org') + self.request_class('https://example.org') + self.request_class('s3://example.org') + self.request_class('ftp://example.org') + self.request_class('about:config') + self.request_class('data:,Hello%2C%20World!') + def test_url_no_scheme(self): self.assertRaises(ValueError, self.request_class, 'foo') self.assertRaises(ValueError, self.request_class, '/foo/') From 9f02df20c51c80739d2362c388930ad93e3de34a Mon Sep 17 00:00:00 2001 From: dswij Date: Wed, 7 Oct 2020 01:10:01 +0700 Subject: [PATCH 347/568] Remove PyDispatcher from general requirements --- setup.py | 2 +- tox.ini | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 0c2281400..c046684c0 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,6 @@ install_requires = [ 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'PyDispatcher>=2.0.5', 'pyOpenSSL>=16.2.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', @@ -38,6 +37,7 @@ extras_require = {} if has_environment_marker_platform_impl_support(): extras_require[':platform_python_implementation == "CPython"'] = [ 'lxml>=3.5.0', + 'PyDispatcher>=2.0.5', ] extras_require[':platform_python_implementation == "PyPy"'] = [ # Earlier lxml versions are affected by diff --git a/tox.ini b/tox.ini index 4e6ca3ec9..d29cea0fc 100644 --- a/tox.ini +++ b/tox.ini @@ -72,7 +72,6 @@ deps = itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - PyDispatcher==2.0.5 #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>= pyOpenSSL==18.0.0 queuelib==1.4.2 @@ -89,7 +88,12 @@ deps = [testenv:pinned] deps = {[pinned]deps} - lxml==3.5.0 + PyDispatcher==2.0.5 + +[testenv: pypy-pinned] +deps = + {[pinned]deps} + PyPyDispatcher==2.1.0 [testenv:windows-pinned] basepython = python3 @@ -111,13 +115,16 @@ commands = [testenv:asyncio-pinned] commands = {[testenv:asyncio]commands} -deps = {[testenv:pinned]deps} +deps = + {[testenv:pinned]deps} + lxml==3.5.0 [testenv:pypy3] basepython = pypy3 deps = {[testenv]deps} - PyDispatcher==2.0.5 + PyPyDispatcher>=2.1.0 + lxml==4.0.0 commands = py.test {posargs:--durations=10 docs scrapy tests} From 2e734e6b35f686f62dea11e9484c7646c7c49ca8 Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Tue, 6 Oct 2020 19:51:05 +0100 Subject: [PATCH 348/568] Minor update on the import order in scrapy/downloadermiddlewares/httpcompression.py --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 56421a6ba..f504302e2 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,5 +1,5 @@ -import zlib import io +import zlib from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse From 156bb0a1d413c7c6acafc30003ef98030a06e47f Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Tue, 6 Oct 2020 19:53:40 +0100 Subject: [PATCH 349/568] Fixing the minor typo on test file path in tests/test_downloadermiddleware_httpcompression.py --- tests/test_downloadermiddleware_httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 7e9f9cc5d..4c5bfc577 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -25,7 +25,7 @@ FORMAT = { # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin 'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin - 'zstd-streaming-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), + 'zstd-streaming-no-content-size': ('html-zstd-streaming-no-content-size.bin', 'zstd'), } From 1a597d5e3dda7a467d69cff51df7e731f2f2d6b5 Mon Sep 17 00:00:00 2001 From: OfirD1 Date: Tue, 6 Oct 2020 21:54:42 +0300 Subject: [PATCH 350/568] moved the sentence about processing pending requests when a spider is closed onto a generic note. --- docs/topics/extensions.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 14096ada4..519f18b63 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -257,6 +257,12 @@ settings: * :setting:`CLOSESPIDER_PAGECOUNT` * :setting:`CLOSESPIDER_ERRORCOUNT` +.. note:: + + When a certain closing condition is met, requests which are + currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` + requests) are still processed. + .. setting:: CLOSESPIDER_TIMEOUT CLOSESPIDER_TIMEOUT @@ -279,8 +285,6 @@ Default: ``0`` An integer which specifies a number of items. If the spider scrapes more than that amount and those items are passed by the item pipeline, the spider will be closed with the reason ``closespider_itemcount``. -Requests which are currently in the downloader queue (up to -:setting:`CONCURRENT_REQUESTS` requests) are still processed. If zero (or non set), spiders won't be closed by number of passed items. .. setting:: CLOSESPIDER_PAGECOUNT From 9461414b14a7444dd52eda517dc5e373f79c2b7e Mon Sep 17 00:00:00 2001 From: dswij Date: Wed, 7 Oct 2020 11:26:53 +0700 Subject: [PATCH 351/568] minor changes to remove unnecessary lines --- .travis.yml | 1 - tox.ini | 19 +++++++------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index fb4802070..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,6 @@ install: tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - $HOME/virtualenvs/$PYPY_VERSION/bin/pypy -m ensurepip --default-pip fi - pip install -U tox twine wheel codecov diff --git a/tox.ini b/tox.ini index d29cea0fc..5dc99621e 100644 --- a/tox.ini +++ b/tox.ini @@ -72,12 +72,14 @@ deps = itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 + !pypy3: PyDispatcher==2.0.5 #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>= pyOpenSSL==18.0.0 queuelib==1.4.2 service_identity==16.0.0 Twisted==20.3.0 w3lib==1.17.0 + #zope.interface==5.1.2 to resolve conflict with Twisted==20.3.0 zope.interface==5.1.2 -rtests/requirements-py3.txt # Extras @@ -88,13 +90,8 @@ deps = [testenv:pinned] deps = {[pinned]deps} - PyDispatcher==2.0.5 - -[testenv: pypy-pinned] -deps = - {[pinned]deps} - PyPyDispatcher==2.1.0 - + lxml==3.5.0 + [testenv:windows-pinned] basepython = python3 deps = @@ -114,26 +111,24 @@ commands = {[testenv]commands} --reactor=asyncio [testenv:asyncio-pinned] +deps = {[testenv:pinned]deps} commands = {[testenv:asyncio]commands} -deps = - {[testenv:pinned]deps} - lxml==3.5.0 [testenv:pypy3] basepython = pypy3 deps = {[testenv]deps} - PyPyDispatcher>=2.1.0 lxml==4.0.0 commands = py.test {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython} -commands = {[testenv:pypy3]commands} deps = {[pinned]deps} lxml==4.0.0 + PyPyDispatcher==2.1.0 +commands = {[testenv:pypy3]commands} [docs] changedir = docs From 13ae17aecc632dc03af1bd8b2b63e48301147cc2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 8 Oct 2020 14:04:52 -0300 Subject: [PATCH 352/568] Add xfail_strict=true to pytest.ini --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index ca8191f42..1c95f715a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,5 @@ [pytest] +xfail_strict = true usefixtures = chdir python_files=test_*.py __init__.py python_classes= @@ -40,4 +41,3 @@ flake8-ignore = scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 tests/test_loader.py E741 - From b55c911ddc21a41a6e4204aaf239a16e892de7b5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 21 Sep 2020 10:59:55 -0300 Subject: [PATCH 353/568] Fix CachingHostnameResolver --- scrapy/resolver.py | 57 +++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f191deac6..0350c82b9 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,4 +1,5 @@ from twisted.internet import defer +from twisted.internet._resolver import HostResolution from twisted.internet.base import ThreadedResolver from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple from zope.interface.declarations import implementer, provider @@ -50,6 +51,27 @@ class CachingThreadedResolver(ThreadedResolver): return result +@provider(IResolutionReceiver) +class _CachingResolutionReceiver: + def __init__(self, resolutionReceiver, hostName): + self.resolutionReceiver = resolutionReceiver + self.hostName = hostName + self.addresses = [] + + def resolutionBegan(self, resolution): + self.resolutionReceiver.resolutionBegan(resolution) + self.resolution = resolution + + def addressResolved(self, address): + self.resolutionReceiver.addressResolved(address) + self.addresses.append(address) + + def resolutionComplete(self): + self.resolutionReceiver.resolutionComplete() + if self.addresses: + dnscache[self.hostName] = self.addresses + + @implementer(IHostnameResolver) class CachingHostnameResolver: """ @@ -73,33 +95,22 @@ class CachingHostnameResolver: def install_on_reactor(self): self.reactor.installNameResolver(self) - def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, - addressTypes=None, transportSemantics='TCP'): - - @provider(IResolutionReceiver) - class CachingResolutionReceiver(resolutionReceiver): - - def resolutionBegan(self, resolution): - super().resolutionBegan(resolution) - self.resolution = resolution - self.resolved = False - - def addressResolved(self, address): - super().addressResolved(address) - self.resolved = True - - def resolutionComplete(self): - super().resolutionComplete() - if self.resolved: - dnscache[hostName] = self.resolution - + def resolveHostName( + self, resolutionReceiver, hostName, portNumber=0, addressTypes=None, transportSemantics="TCP" + ): try: - return dnscache[hostName] + addresses = dnscache[hostName] except KeyError: return self.original_resolver.resolveHostName( - CachingResolutionReceiver(), + _CachingResolutionReceiver(resolutionReceiver, hostName), hostName, portNumber, addressTypes, - transportSemantics + transportSemantics, ) + else: + resolutionReceiver.resolutionBegan(HostResolution(hostName)) + for addr in addresses: + resolutionReceiver.addressResolved(addr) + resolutionReceiver.resolutionComplete() + return resolutionReceiver From 8fe5876597825a4df03bd15e6b1cd1682b806de8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 30 Sep 2020 14:56:17 -0300 Subject: [PATCH 354/568] HostResolution implementation --- scrapy/resolver.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 0350c82b9..0bef555a6 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,7 +1,6 @@ from twisted.internet import defer -from twisted.internet._resolver import HostResolution from twisted.internet.base import ThreadedResolver -from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple +from twisted.internet.interfaces import IHostResolution, IHostnameResolver, IResolutionReceiver, IResolverSimple from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache @@ -51,6 +50,15 @@ class CachingThreadedResolver(ThreadedResolver): return result +@implementer(IHostResolution) +class HostResolution: + def __init__(self, name): + self.name = name + + def cancel(self): + raise NotImplementedError() + + @provider(IResolutionReceiver) class _CachingResolutionReceiver: def __init__(self, resolutionReceiver, hostName): From 868826b346714ceff821886536a3b259072f8396 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 2 Oct 2020 15:16:58 -0300 Subject: [PATCH 355/568] CachingHostnameResolver tests --- .../alternative_name_resolver.py | 15 ---------- .../caching_hostname_resolver.py | 30 +++++++++++++++++++ .../caching_hostname_resolver_ipv6.py | 19 ++++++++++++ tests/CrawlerProcess/default_name_resolver.py | 11 +++++-- tests/test_crawler.py | 20 +++++++++---- 5 files changed, 72 insertions(+), 23 deletions(-) delete mode 100644 tests/CrawlerProcess/alternative_name_resolver.py create mode 100644 tests/CrawlerProcess/caching_hostname_resolver.py create mode 100644 tests/CrawlerProcess/caching_hostname_resolver_ipv6.py diff --git a/tests/CrawlerProcess/alternative_name_resolver.py b/tests/CrawlerProcess/alternative_name_resolver.py deleted file mode 100644 index 2c466da04..000000000 --- a/tests/CrawlerProcess/alternative_name_resolver.py +++ /dev/null @@ -1,15 +0,0 @@ -import scrapy -from scrapy.crawler import CrawlerProcess - - -class IPv6Spider(scrapy.Spider): - name = "ipv6_spider" - start_urls = ["http://[::1]"] - - -process = CrawlerProcess(settings={ - "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", -}) -process.crawl(IPv6Spider) -process.start() diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py new file mode 100644 index 000000000..f9eab3543 --- /dev/null +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -0,0 +1,30 @@ +import sys + +import scrapy +from scrapy.crawler import CrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution) + """ + name = "caching_hostname_resolver_spider" + + def start_requests(self): + yield scrapy.Request(self.url) + + def parse(self, response): + for _ in range(10): + yield scrapy.Request(response.url, dont_filter=True, callback=self.ignore_response) + + def ignore_response(self, response): + self.logger.info(repr(response.ip_address)) + + +if __name__ == "__main__": + process = CrawlerProcess(settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + }) + process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) + process.start() diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py new file mode 100644 index 000000000..3340d2f84 --- /dev/null +++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py @@ -0,0 +1,19 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes without a twisted.internet.error.DNSLookupError exception + """ + name = "caching_hostname_resolver_spider" + start_urls = ["http://[::1]"] + + +if __name__ == "__main__": + process = CrawlerProcess(settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + }) + process.crawl(CachingHostnameResolverSpider) + process.start() diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py index 60d91b68b..05a98fbec 100644 --- a/tests/CrawlerProcess/default_name_resolver.py +++ b/tests/CrawlerProcess/default_name_resolver.py @@ -3,10 +3,15 @@ from scrapy.crawler import CrawlerProcess class IPv6Spider(scrapy.Spider): + """ + Raises a twisted.internet.error.DNSLookupError: + the default name resolver does not handle IPv6 addresses. + """ name = "ipv6_spider" start_urls = ["http://[::1]"] -process = CrawlerProcess(settings={"RETRY_ENABLED": False}) -process.crawl(IPv6Spider) -process.start() +if __name__ == "__main__": + process = CrawlerProcess(settings={"RETRY_ENABLED": False}) + process.crawl(IPv6Spider) + process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 85035a220..246e54860 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -22,6 +22,8 @@ from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv +from tests.mockserver import MockServer + class BaseCrawlerTest(unittest.TestCase): @@ -280,9 +282,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: - def run_script(self, script_name): + def run_script(self, script_name, *script_args): script_path = os.path.join(self.script_dir, script_name) - args = (sys.executable, script_path) + args = [sys.executable, script_path] + list(script_args) p = subprocess.Popen(args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() @@ -321,11 +323,19 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log) - def test_ipv6_alternative_name_resolver(self): - log = self.run_script('alternative_name_resolver.py') - self.assertIn('Spider closed (finished)', log) + def test_caching_hostname_resolver_ipv6(self): + log = self.run_script("caching_hostname_resolver_ipv6.py") + self.assertIn("Spider closed (finished)", log) self.assertNotIn("twisted.internet.error.DNSLookupError", log) + def test_caching_hostname_resolver_finite_execution(self): + with MockServer() as mock_server: + log = self.run_script("caching_hostname_resolver.py", mock_server.http_address) + self.assertIn("Spider closed (finished)", log) + self.assertNotIn("ERROR: Error downloading", log) + self.assertNotIn("TimeoutError", log) + self.assertNotIn("twisted.internet.error.DNSLookupError", log) + def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") self.assertIn("Spider closed (finished)", log) From 015c82b974841897a637dcd3ddbd40ab48a70ecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 11 Oct 2020 22:09:45 +0200 Subject: [PATCH 356/568] Scrapy 2.4 release notes (#4808) --- docs/news.rst | 306 ++++++++++++++++++++++++++++++++- docs/topics/asyncio.rst | 2 + docs/topics/feed-exports.rst | 18 +- docs/topics/media-pipeline.rst | 31 ++-- docs/topics/settings.rst | 2 +- 5 files changed, 344 insertions(+), 15 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 850b323ef..d5c342c86 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,310 @@ Release notes ============= +.. _release-2.4.0: + +Scrapy 2.4.0 (2020-10-??) +------------------------- + +Highlights: + +* Python 3.5 support has been dropped. + +* The ``file_path`` method of :ref:`media pipelines ` + can now access the source :ref:`item `. + + This allows you to set a download file path based on item data. + +* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows + to define keyword parameters to pass to :ref:`item exporter classes + ` + +* You can now choose whether :ref:`feed exports ` + overwrite or append to the output file. + + For example, when using the :command:`crawl` or :command:`runspider` + commands, you can use the ``-O`` option instead of ``-o`` to overwrite the + output file. + +* Zstd-compressed responses are now supported if zstandard_ is installed. + +* In settings, where the import path of a class is required, it is now + possible to pass a class object instead. + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +* Python 3.6 or greater is now required; support for Python 3.5 has been + dropped + + As a result: + + - When using PyPy, PyPy 7.2.0 or greater :ref:`is now required + ` + + - For Amazon S3 storage support in :ref:`feed exports + ` or :ref:`media pipelines + `, botocore_ 1.4.87 or greater is now required + + - To use the :ref:`images pipeline `, Pillow_ 4.0.0 or + greater is now required + + (:issue:`4718`, :issue:`4732`, :issue:`4733`, :issue:`4742`, :issue:`4743`, + :issue:`4764`) + + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again + discards cookies defined in :attr:`Request.headers + `. + + We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it + was reported that the current implementation could break existing code. + + If you need to set cookies for a request, use the :class:`Request.cookies + ` parameter. + + A future version of Scrapy will include a new, better implementation of the + reverted bug fix. + + (:issue:`4717`, :issue:`4823`) + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* :class:`scrapy.extensions.feedexport.S3FeedStorage` no longer reads the + values of ``access_key`` and ``secret_key`` from the running project + settings when they are not passed to its ``__init__`` method; you must + either pass those parameters to its ``__init__`` method or use + :class:`S3FeedStorage.from_crawler + ` + (:issue:`4356`, :issue:`4411`, :issue:`4688`) + +* :attr:`Rule.process_request ` + no longer admits callables which expect a single ``request`` parameter, + rather than both ``request`` and ``response`` (:issue:`4818`) + + +Deprecations +~~~~~~~~~~~~ + +* In custom :ref:`media pipelines `, signatures that + do not accept a keyword-only ``item`` parameter in any of the methods that + :ref:`now support this parameter ` are now + deprecated (:issue:`4628`, :issue:`4686`) + +* In custom :ref:`feed storage backend classes `, + ``__init__`` method signatures that do not accept a keyword-only + ``feed_options`` parameter are now deprecated (:issue:`547`, :issue:`716`, + :issue:`4512`) + +* The :class:`scrapy.utils.python.WeakKeyCache` class is now deprecated + (:issue:`4684`, :issue:`4701`) + +* The :func:`scrapy.utils.boto.is_botocore` function is now deprecated, use + :func:`scrapy.utils.boto.is_botocore_available` instead (:issue:`4734`, + :issue:`4776`) + + +New features +~~~~~~~~~~~~ + +.. _media-pipeline-item-parameter: + +* The following methods of :ref:`media pipelines ` now + accept an ``item`` keyword-only parameter containing the source + :ref:`item `: + + - In :class:`scrapy.pipelines.files.FilesPipeline`: + + - :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.file_path` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_to_download` + + - In :class:`scrapy.pipelines.images.ImagesPipeline`: + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.file_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.file_path` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.get_images` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.media_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.media_to_download` + + (:issue:`4628`, :issue:`4686`) + +* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows + to define keyword parameters to pass to :ref:`item exporter classes + ` (:issue:`4606`, :issue:`4768`) + +* :ref:`Feed exports ` gained overwrite support: + + * When using the :command:`crawl` or :command:`runspider` commands, you + can use the ``-O`` option instead of ``-o`` to overwrite the output + file + + * You can use the ``overwrite`` key in the :setting:`FEEDS` setting to + configure whether to overwrite the output file (``True``) or append to + its content (``False``) + + * The ``__init__`` and ``from_crawler`` methods of :ref:`feed storage + backend classes ` now receive a new keyword-only + parameter, ``feed_options``, which is a dictionary of :ref:`feed + options ` + + (:issue:`547`, :issue:`716`, :issue:`4512`) + +* Zstd-compressed responses are now supported if zstandard_ is installed + (:issue:`4831`) + +* In settings, where the import path of a class is required, it is now + possible to pass a class object instead (:issue:`3870`, :issue:`3873`). + + This includes also settings where only part of its value is made of an + import path, such as :setting:`DOWNLOADER_MIDDLEWARES` or + :setting:`DOWNLOAD_HANDLERS`. + +* :ref:`Downloader middlewares ` can now + override :class:`response.request `. + + If a :ref:`downloader middleware ` returns + a :class:`~scrapy.http.Response` object from + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` + or + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception` + with a custom :class:`~scrapy.http.Request` object assigned to + :class:`response.request `: + + - The response is handled by the callback of that custom + :class:`~scrapy.http.Request` object, instead of being handled by the + callback of the original :class:`~scrapy.http.Request` object + + - That custom :class:`~scrapy.http.Request` object is now sent as the + ``request`` argument to the :signal:`response_received` signal, instead + of the original :class:`~scrapy.http.Request` object + + (:issue:`4529`, :issue:`4632`) + +* When using the :ref:`FTP feed storage backend `: + + - It is now possible to set the new ``overwrite`` :ref:`feed option + ` to ``False`` to append to an existing file instead of + overwriting it + + - The FTP password can now be omitted if it is not necessary + + (:issue:`547`, :issue:`716`, :issue:`4512`) + +* The ``__init__`` method of :class:`~scrapy.exporters.CsvItemExporter` now + supports an ``errors`` parameter to indicate how to handle encoding errors + (:issue:`4755`) + +* When :ref:`using asyncio `, it is now possible to + :ref:`set a custom asyncio loop ` (:issue:`4306`, + :issue:`4414`) + +* Serialized requests (see :ref:`topics-jobs`) now support callbacks that are + spider methods that delegate on other callable (:issue:`4756`) + +* When a response is larger than :setting:`DOWNLOAD_MAXSIZE`, the logged + message is now a warning, instead of an error (:issue:`3874`, + :issue:`3886`, :issue:`4752`) + + +Bug fixes +~~~~~~~~~ + +* The :command:`genspider` command no longer overwrites existing files + unless the ``--force`` option is used (:issue:`4561`, :issue:`4616`, + :issue:`4623`) + +* Cookies with an empty value are no longer considered invalid cookies + (:issue:`4772`) + +* The :command:`runspider` command now supports files with the ``.pyw`` file + extension (:issue:`4643`, :issue:`4646`) + +* The :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` + middleware now simply ignores unsupported proxy values (:issue:`3331`, + :issue:`4778`) + +* Checks for generator callbacks with a ``return`` statement no longer warn + about ``return`` statements in nested functions (:issue:`4720`, + :issue:`4721`) + +* The system file mode creation mask no longer affects the permissions of + files generated using the :command:`startproject` command (:issue:`4722`) + +* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names + (:issue:`861`, :issue:`4746`) + +* :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can + work when using a headless browser (:issue:`4835`) + + +Documentation +~~~~~~~~~~~~~ + +* The :setting:`FEED_URI_PARAMS` setting is now documented (:issue:`4671`, + :issue:`4724`) + +* Improved the documentation of + :ref:`link extractors ` with an usage example from + a spider callback and reference documentation for the + :class:`~scrapy.link.Link` class (:issue:`4751`, :issue:`4775`) + +* Clarified the impact of :setting:`CONCURRENT_REQUESTS` when using the + :class:`~scrapy.extensions.closespider.CloseSpider` extension + (:issue:`4836`) + +* Removed references to Python 2’s ``unicode`` type (:issue:`4547`, + :issue:`4703`) + +* We now have an :ref:`official deprecation policy ` + (:issue:`4705`) + +* Our :ref:`documentation policies ` now cover usage + of Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged` + directives, and we have removed usages referencing Scrapy 1.4.0 and earlier + versions (:issue:`3971`, :issue:`4310`) + +* Other documentation cleanups (:issue:`4090`, :issue:`4782`, :issue:`4800`, + :issue:`4801`, :issue:`4809`, :issue:`4816`, :issue:`4825`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Extended typing hints (:issue:`4243`, :issue:`4691`) + +* Added tests for the :command:`check` command (:issue:`4663`) + +* Fixed test failures on Debian (:issue:`4726`, :issue:`4727`, :issue:`4735`) + +* Improved Windows test coverage (:issue:`4723`) + +* Switched to :ref:`formatted string literals ` where possible + (:issue:`4307`, :issue:`4324`, :issue:`4672`) + +* Modernized :func:`super` usage (:issue:`4707`) + +* Other code and test cleanups (:issue:`1790`, :issue:`3288`, :issue:`4165`, + :issue:`4564`, :issue:`4651`, :issue:`4714`, :issue:`4738`, :issue:`4745`, + :issue:`4747`, :issue:`4761`, :issue:`4765`, :issue:`4804`, :issue:`4817`, + :issue:`4820`, :issue:`4822`, :issue:`4839`) + + .. _release-2.3.0: Scrapy 2.3.0 (2020-08-04) @@ -4008,9 +4312,9 @@ First release of Scrapy. .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ .. _Twisted: https://twistedmatrix.com/trac/ -.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _w3lib: https://github.com/scrapy/w3lib .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py .. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 .. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/ .. _Zsh: https://www.zsh.org/ +.. _zstandard: https://pypi.org/project/zstandard/ diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index bfb430d52..91e1cca0d 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -1,3 +1,5 @@ +.. _using-asyncio: + ======= asyncio ======= diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 1744cfd74..843ed25f9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -184,7 +184,7 @@ The feeds are stored on `Amazon S3`_. * ``s3://mybucket/path/to/export.csv`` * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` - * Required external libraries: `botocore`_ + * Required external libraries: `botocore`_ >= 1.4.87 The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: @@ -319,6 +319,8 @@ For instance:: }, } +.. _feed-options: + The following is a list of the accepted keys and the setting that is used as a fallback value if that key is not provided for a specific feed definition: @@ -329,6 +331,8 @@ as a fallback value if that key is not provided for a specific feed definition: - ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + .. versionadded:: 2.3.0 + - ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`. - ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. @@ -337,6 +341,8 @@ as a fallback value if that key is not provided for a specific feed definition: - ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. + .. versionadded:: 2.4.0 + - ``overwrite``: whether to overwrite the file if it already exists (``True``) or append to its content (``False``). @@ -355,6 +361,8 @@ as a fallback value if that key is not provided for a specific feed definition: - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) + .. versionadded:: 2.4.0 + - ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`. - ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. @@ -517,7 +525,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. setting:: FEED_EXPORT_BATCH_ITEM_COUNT FEED_EXPORT_BATCH_ITEM_COUNT ------------------------------ +---------------------------- + +.. versionadded:: 2.3.0 Default: ``0`` @@ -586,11 +596,15 @@ The function signature should be as follows: If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id`` is always ``1``. + .. versionadded:: 2.3.0 + - ``batch_time``: UTC date and time, in ISO format with ``:`` replaced with ``-``. See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + .. versionadded:: 2.3.0 + - ``time``: ``batch_time``, with microseconds set to ``0``. :type params: dict diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 06809c24b..156897274 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -56,6 +56,8 @@ this: error will be logged and the file won't be present in the ``files`` field. +.. _images-pipeline: + Using the Images Pipeline ========================= @@ -68,14 +70,10 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to -JPEG/RGB format, so you need to install this library in order to use it. -`Python Imaging Library`_ (PIL) should also work in most cases, but it is known -to cause troubles in some setups, so we recommend to use `Pillow`_ instead of -PIL. +The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for +thumbnailing and normalizing images to JPEG/RGB format. .. _Pillow: https://github.com/python-pillow/Pillow -.. _Python Imaging Library: http://www.pythonware.com/products/pil/ .. _topics-media-pipeline-enabling: @@ -164,14 +162,17 @@ FTP supports two different connection modes: active or passive. Scrapy uses the passive connection mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. +.. _media-pipelines-s3: + Amazon S3 storage ----------------- .. setting:: FILES_STORE_S3_ACL .. setting:: IMAGES_STORE_S3_ACL -:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3 -bucket. Scrapy will automatically upload the files to the bucket. +If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and +:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will +automatically upload the files to the bucket. For example, this is a valid :setting:`IMAGES_STORE` value:: @@ -187,8 +188,9 @@ policy:: For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. -Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like -self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings:: +You can also use other S3-like storages. Storages like self-hosted `Minio`_ or +`s3.scality`_. All you need to do is set endpoint option in you Scrapy +settings:: AWS_ENDPOINT_URL = 'http://minio.example.com:9000' @@ -197,9 +199,10 @@ For self-hosting you also might feel the need not to use SSL and not to verify S AWS_USE_SSL = False # or True (None by default) AWS_VERIFY = False # or True (None by default) +.. _botocore: https://github.com/boto/botocore +.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _Minio: https://github.com/minio/minio .. _s3.scality: https://s3.scality.com/ -.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _media-pipeline-gcs: @@ -446,6 +449,9 @@ See here the methods that you can override in your custom Files Pipeline: By default the :meth:`file_path` method returns ``full/.``. + .. versionadded:: 2.4 + The *item* parameter. + .. method:: FilesPipeline.get_media_requests(item, info) As seen on the workflow, the pipeline will get the URLs of the images to @@ -582,6 +588,9 @@ See here the methods that you can override in your custom Images Pipeline: By default the :meth:`file_path` method returns ``full/.``. + .. versionadded:: 2.4 + The *item* parameter. + .. method:: ImagesPipeline.get_media_requests(item, info) Works the same way as :meth:`FilesPipeline.get_media_requests` method, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 71331c841..912757850 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -102,7 +102,7 @@ module and documented in the :ref:`topics-settings-ref` section. Import paths and classes ======================== -.. versionadded:: VERSION +.. versionadded:: 2.4.0 When a setting references a callable object to be imported by Scrapy, such as a class or a function, there are two different ways you can specify that object: From 47eac8374040c0dc389eb8152a60938dab358bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 11 Oct 2020 22:11:14 +0200 Subject: [PATCH 357/568] Set a release date for Scrapy 2.4.0 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index d5c342c86..a3889705d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.4.0: -Scrapy 2.4.0 (2020-10-??) +Scrapy 2.4.0 (2020-10-11) ------------------------- Highlights: From c340e72988fc6ec615b7b9851c3d28c16c26a839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 11 Oct 2020 22:12:45 +0200 Subject: [PATCH 358/568] =?UTF-8?q?Bump=20version:=202.3.0=20=E2=86=92=202?= =?UTF-8?q?.4.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3c1c8f891..0f142472e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.3.0 +current_version = 2.4.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 276cbf9e2..197c4d5c2 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.3.0 +2.4.0 From fd663fd4ad5a69ba0403a2eec5bdc29a0109b0d4 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Tue, 13 Oct 2020 18:35:06 +0300 Subject: [PATCH 359/568] __init__ stats parameter - optional, stats==None - covered. --- scrapy/downloadermiddlewares/httpcompression.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index ca80e9444..8980e9ca4 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -18,7 +18,7 @@ except ImportError: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, stats): + def __init__(self, stats=None): self.stats = stats @classmethod @@ -40,8 +40,9 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) - self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) - self.stats.inc_value('httpcompression/response_count', spider=spider) + if self.stats: + self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + self.stats.inc_value('httpcompression/response_count', spider=spider) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) From d32d0d27393ce55490c44d9fb039130320461865 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Tue, 13 Oct 2020 18:36:41 +0300 Subject: [PATCH 360/568] testcase added for HttpCompressionMiddleware with no stats --- tests/test_downloadermiddleware_httpcompression.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 144fd3378..5f8e76e0a 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -29,7 +29,7 @@ class HttpCompressionTest(TestCase): def setUp(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider('scrapytest.org') - self.mw = HttpCompressionMiddleware(self.crawler.stats) + self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) self.crawler.stats.open_spider(self.spider) def _getresponse(self, coding): @@ -79,6 +79,18 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual('httpcompression/response_count', 1) self.assertStatsEqual('httpcompression/response_bytes', 74837) + def test_process_response_gzip_no_stats(self): + mw = HttpCompressionMiddleware() + response = self._getresponse('gzip') + request = response.request + + self.assertEqual(response.headers['Content-Encoding'], b'gzip') + newresponse = mw.process_response(request, response, self.spider) + self.assertEqual(mw.stats, None) + assert newresponse is not response + assert newresponse.body.startswith(b' Date: Tue, 13 Oct 2020 18:41:58 +0300 Subject: [PATCH 361/568] testcase added for COMPRESSION_ENABLED setting --- ...st_downloadermiddleware_httpcompression.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5f8e76e0a..c2bcbd63c 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -6,6 +6,7 @@ from gzip import GzipFile from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS +from scrapy.exceptions import NotConfigured from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler @@ -60,6 +61,27 @@ class HttpCompressionTest(TestCase): str(self.crawler.stats.get_stats(self.spider)) ) + def test_setting_false_compression_enabled(self): + self.assertRaises( + NotConfigured, + HttpCompressionMiddleware.from_crawler, + get_crawler(settings_dict={'COMPRESSION_ENABLED': False}) + ) + + def test_setting_default_compression_enabled(self): + self.assertIsInstance( + HttpCompressionMiddleware.from_crawler(get_crawler()), + HttpCompressionMiddleware + ) + + def test_setting_true_compression_enabled(self): + self.assertIsInstance( + HttpCompressionMiddleware.from_crawler( + get_crawler(settings_dict={'COMPRESSION_ENABLED': True}) + ), + HttpCompressionMiddleware + ) + def test_process_request(self): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers From 585e4a8aee649f2b439c42d91d857ed1fbdf4fa5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 9 Oct 2020 10:41:19 -0300 Subject: [PATCH 362/568] Replace local server address --- tests/test_crawler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 246e54860..b6de33189 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -330,7 +330,8 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_caching_hostname_resolver_finite_execution(self): with MockServer() as mock_server: - log = self.run_script("caching_hostname_resolver.py", mock_server.http_address) + http_address = mock_server.http_address.replace("0.0.0.0", "127.0.0.1") + log = self.run_script("caching_hostname_resolver.py", http_address) self.assertIn("Spider closed (finished)", log) self.assertNotIn("ERROR: Error downloading", log) self.assertNotIn("TimeoutError", log) From 7187247c01d3629000b8b460eea26064043c5595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 Oct 2020 10:23:59 +0100 Subject: [PATCH 363/568] Add PyDispatcher>=2.0.5 back to dependencies for old pip --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index c046684c0..aa75c7a34 100644 --- a/setup.py +++ b/setup.py @@ -33,12 +33,12 @@ install_requires = [ 'itemadapter>=0.1.0', ] extras_require = {} - +cpython_dependencies = [ + 'lxml>=3.5.0', + 'PyDispatcher>=2.0.5', +] if has_environment_marker_platform_impl_support(): - extras_require[':platform_python_implementation == "CPython"'] = [ - 'lxml>=3.5.0', - 'PyDispatcher>=2.0.5', - ] + extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies extras_require[':platform_python_implementation == "PyPy"'] = [ # Earlier lxml versions are affected by # https://foss.heptapod.net/pypy/pypy/-/issues/2498, @@ -49,7 +49,7 @@ if has_environment_marker_platform_impl_support(): 'PyPyDispatcher>=2.1.0', ] else: - install_requires.append('lxml>=3.5.0') + install_requires.extend(cpython_dependencies) setup( From 13bcdc9f881727f5a140d4b8671c81c7de3bc572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 Oct 2020 10:28:42 +0100 Subject: [PATCH 364/568] Restore pinned dependencies in tox.ini --- tox.ini | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tox.ini b/tox.ini index 5dc99621e..b90ed867f 100644 --- a/tox.ini +++ b/tox.ini @@ -66,21 +66,17 @@ commands = [pinned] deps = -ctests/constraints.txt - #using cryptography-2.3.1 to solve dependencies conflict on pip20.2>= - cryptography==2.3.1 + cryptography==2.0 cssselect==0.9.1 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - !pypy3: PyDispatcher==2.0.5 - #using pyOpenSSL<18.1 to solve dependencies conflict with mitmproxy on pip20.2>= - pyOpenSSL==18.0.0 + pyOpenSSL==16.2.0 queuelib==1.4.2 service_identity==16.0.0 - Twisted==20.3.0 + Twisted==17.9.0 w3lib==1.17.0 - #zope.interface==5.1.2 to resolve conflict with Twisted==20.3.0 - zope.interface==5.1.2 + zope.interface==4.1.3 -rtests/requirements-py3.txt # Extras botocore==1.4.87 @@ -91,6 +87,7 @@ deps = deps = {[pinned]deps} lxml==3.5.0 + PyDispatcher==2.0.5 [testenv:windows-pinned] basepython = python3 @@ -99,6 +96,7 @@ deps = # First lxml version that includes a Windows wheel for Python 3.6, so we do # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 + PyDispatcher==2.0.5 [testenv:extra-deps] deps = From 3e5bc7773732aa7e9839934cc45cac8252a36aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 Oct 2020 10:31:33 +0100 Subject: [PATCH 365/568] Remove pinning from the PyPy job --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index b90ed867f..878395e6f 100644 --- a/tox.ini +++ b/tox.ini @@ -114,9 +114,6 @@ commands = {[testenv:asyncio]commands} [testenv:pypy3] basepython = pypy3 -deps = - {[testenv]deps} - lxml==4.0.0 commands = py.test {posargs:--durations=10 docs scrapy tests} From a5872a0fad090c3d3f91f6e03a772265aa50f901 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 30 Oct 2020 20:36:39 +0200 Subject: [PATCH 366/568] Fix output file overwrite with -O (FeedExporter updated) (#4859) --- scrapy/extensions/feedexport.py | 2 +- tests/test_commands.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 7dcb2f52e..3fb4d0e2c 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -452,7 +452,7 @@ class FeedExporter: crawler = getattr(self, 'crawler', None) def build_instance(builder, *preargs): - return build_storage(builder, uri, preargs=preargs) + return build_storage(builder, uri, feed_options=feed_options, preargs=preargs) if crawler and hasattr(feedcls, 'from_crawler'): instance = build_instance(feedcls.from_crawler, crawler) diff --git a/tests/test_commands.py b/tests/test_commands.py index 3e54a0948..85aee55a5 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -680,9 +680,14 @@ class MySpider(scrapy.Spider): ) return [] """ + with open(os.path.join(self.cwd, "example.json"), "w") as f1: + f1.write("not empty") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + with open(os.path.join(self.cwd, "example.json")) as f2: + first_line = f2.readline() + self.assertNotEqual(first_line, "not empty") def test_output_and_overwrite_output(self): spider_code = """ @@ -813,9 +818,14 @@ class MySpider(scrapy.Spider): ) return [] """ + with open(os.path.join(self.cwd, "example.json"), "w") as f1: + f1.write("not empty") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + with open(os.path.join(self.cwd, "example.json")) as f2: + first_line = f2.readline() + self.assertNotEqual(first_line, "not empty") def test_output_and_overwrite_output(self): spider_code = """ From e9c3188189cffc965797b1b77fc5dc5cfa06b5cb Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 30 Oct 2020 21:23:29 +0200 Subject: [PATCH 367/568] Update scrapy/downloadermiddlewares/httpcompression.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/downloadermiddlewares/httpcompression.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 8980e9ca4..87f744956 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -25,7 +25,12 @@ class HttpCompressionMiddleware: def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): raise NotConfigured - return cls(crawler.stats) + try: + return cls(stats=crawler.stats) + except TypeError: + result = cls() + result.stats = crawler.stats + return result def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', From 7327145bf3d45b83b110a46ff79d2e337e36b520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 Oct 2020 21:34:15 +0100 Subject: [PATCH 368/568] Remove mitmproxy from pinned environments --- tests/requirements-py3.txt | 2 -- tests/test_proxy_connect.py | 11 +++++------ tox.ini | 10 +++++++++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2247ed917..d44632d8b 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,8 +1,6 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy; python_version >= '3.7' -mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 9eabe6b49..0db148910 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -5,8 +5,6 @@ import re import sys from subprocess import Popen, PIPE from urllib.parse import urlsplit, urlunsplit -from unittest import skipIf - from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -57,13 +55,14 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) -@skipIf("pypy" in sys.executable, - "mitmproxy does not support PyPy") -@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), - "mitmproxy does not support Windows when running Python < 3.7") class ProxyConnectTestCase(TestCase): def setUp(self): + try: + import mitmproxy + except ImportError: + self.skipTest('mitmproxy is not installed') + self.mockserver = MockServer() self.mockserver.__enter__() self._oldenv = os.environ.copy() diff --git a/tox.ini b/tox.ini index 878395e6f..6149a9c58 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,10 @@ minversion = 1.7.0 deps = -ctests/constraints.txt -rtests/requirements-py3.txt + # mitmproxy does not support PyPy + # mitmproxy does not support Windows when running Python < 3.7 + mitmproxy; python_version >= '3.7' and implementation_name != 'pypy' + mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras boto3>=1.13.0 botocore>=1.4.87 @@ -26,7 +30,7 @@ download = true commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = - pip install --use-feature=2020-resolver {opts} {packages} + pip install --use-feature=2020-resolver {opts} {packages} [testenv:typing] basepython = python3 @@ -78,6 +82,10 @@ deps = w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt + + # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies + # above, hence we do not install it in pinned environments at the moment + # Extras botocore==1.4.87 google-cloud-storage==1.29.0 From 8e7b756727cbd8c207acce3083ae29c85a6c6d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 4 Nov 2020 21:26:55 +0100 Subject: [PATCH 369/568] Solve Flake8-reported issues --- tests/test_proxy_connect.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 0db148910..afdfb2578 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,6 +1,5 @@ import json import os -import platform import re import sys from subprocess import Popen, PIPE @@ -59,7 +58,7 @@ class ProxyConnectTestCase(TestCase): def setUp(self): try: - import mitmproxy + import mitmproxy # noqa: F401 except ImportError: self.skipTest('mitmproxy is not installed') From 906626cf0befe332cb61b9cdda7108d61afec776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 4 Nov 2020 21:50:12 +0100 Subject: [PATCH 370/568] Skip MiddlewareUsingCoro::test_asyncdef on asyncio and old Twisted --- tests/test_downloadermiddleware.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 79f24c8a1..55af4c240 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,11 +1,12 @@ import asyncio -from unittest import mock +from unittest import mock, SkipTest from pytest import mark from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase from twisted.python.failure import Failure +from twisted.python.versions import Version from scrapy.http import Request, Response from scrapy.spiders import Spider @@ -211,10 +212,24 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.assertFalse(download_func.called) +@mark.usefixtures('reactor_pytest') class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): + import twisted + if ( + self.reactor_pytest == 'asyncio' + and twisted.version < Version('twisted', 18, 4, 0) + ): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'hangs when using AsyncIO and Twisted versions lower than ' + '18.4.0' + ) + + from twisted.python.versions import Version + resp = Response('http://example.com/index.html') class CoroMiddleware: From 6eaf0c5cc99204daed601369c9cea6f7c5e2f878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 4 Nov 2020 21:54:00 +0100 Subject: [PATCH 371/568] Use Ubuntu Bionic for PyPy tests to try to get a newer OpenSSL version recognized --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b883c5b78..f0eafd137 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ matrix: - env: TOXENV=py python: 3.6 - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 + dist: bionic - env: TOXENV=py python: 3.7 From c292957cb19085137146bde72fe82639591c5e1c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 5 Nov 2020 11:15:58 -0300 Subject: [PATCH 372/568] Run Windows tests on GitHub actions (#4869) --- .github/workflows/main.yml | 31 +++++++++++++++++++++++++++++++ azure-pipelines.yml | 22 ---------------------- 2 files changed, 31 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 azure-pipelines.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..28771216c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,31 @@ +name: Run test suite +on: [push, pull_request] + +jobs: + test-windows: + name: "Windows Tests" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest] + python-version: [3.7, 3.8] + env: [TOXENV: py] + include: + - os: windows-latest + python-version: 3.6 + env: + TOXENV: windows-pinned + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Run test suite + env: ${{ matrix.env }} + run: | + pip install -U tox twine wheel codecov + tox diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index c03e258c7..000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,22 +0,0 @@ -variables: - TOXENV: py -pool: - vmImage: 'windows-latest' -strategy: - matrix: - Python36: - python.version: '3.6' - TOXENV: windows-pinned - Python37: - python.version: '3.7' - Python38: - python.version: '3.8' -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - displayName: 'Use Python $(python.version)' -- script: | - pip install -U tox twine wheel codecov - tox - displayName: 'Run test suite' From 5b5478ae9d6f8d5e0028fe47a63252679c457364 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 5 Nov 2020 14:01:34 -0300 Subject: [PATCH 373/568] Call asyncio.get_event_loop when installing the asyncio reactor --- scrapy/utils/reactor.py | 3 +- .../CrawlerProcess/asyncio_deferred_signal.py | 45 +++++++++++++++++++ tests/test_crawler.py | 16 +++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 tests/CrawlerProcess/asyncio_deferred_signal.py diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 831d29462..6723d9b37 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -60,8 +60,9 @@ def install_reactor(reactor_path, event_loop_path=None): if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) else: - event_loop = asyncio.new_event_loop() + event_loop = asyncio.get_event_loop() asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py new file mode 100644 index 000000000..bce300afe --- /dev/null +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -0,0 +1,45 @@ +import asyncio +import sys + +import scrapy + +from scrapy.crawler import CrawlerProcess +from twisted.internet.defer import Deferred + + +class UppercasePipeline: + async def _open_spider(self, spider): + spider.logger.info("async pipeline opened!") + await asyncio.sleep(0.1) + + def open_spider(self, spider): + loop = asyncio.get_event_loop() + return Deferred.fromFuture(loop.create_task(self._open_spider(spider))) + + def process_item(self, item, spider): + return {"url": item["url"].upper()} + + +class UrlSpider(scrapy.Spider): + name = "url_spider" + start_urls = ["data:,"] + custom_settings = { + "ITEM_PIPELINES": {UppercasePipeline: 100}, + } + + def parse(self, response): + yield {"url": response.url} + + +if __name__ == "__main__": + try: + ASYNCIO_EVENT_LOOP = sys.argv[1] + except IndexError: + ASYNCIO_EVENT_LOOP = None + + process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP, + }) + process.crawl(UrlSpider) + process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b6de33189..0faaa79a3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -364,6 +364,22 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) self.assertIn("Using asyncio event loop: uvloop.Loop", log) + @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly") + @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows") + def test_custom_loop_asyncio_deferred_signal(self): + log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + self.assertIn("async pipeline opened!", log) + + def test_default_loop_asyncio_deferred_signal(self): + log = self.run_script("asyncio_deferred_signal.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertNotIn("Using asyncio event loop: uvloop.Loop", log) + self.assertIn("async pipeline opened!", log) + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') From ea851b910e580e840653274182548314e15b4312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 12:34:29 +0100 Subject: [PATCH 374/568] Clean up Twisted version check --- tests/test_downloadermiddleware.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 55af4c240..6168e0176 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -2,6 +2,7 @@ import asyncio from unittest import mock, SkipTest from pytest import mark +from twisted import version as twisted_version from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase @@ -217,10 +218,9 @@ class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): - import twisted if ( self.reactor_pytest == 'asyncio' - and twisted.version < Version('twisted', 18, 4, 0) + and twisted_version < Version('twisted', 18, 4, 0) ): raise SkipTest( 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' @@ -228,8 +228,6 @@ class MiddlewareUsingCoro(ManagerTestCase): '18.4.0' ) - from twisted.python.versions import Version - resp = Response('http://example.com/index.html') class CoroMiddleware: From fea5a118993855808183b32809c2c304db28c364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 12:59:46 +0100 Subject: [PATCH 375/568] Also skip test_asyncdef_asyncio on old Twisted versions --- tests/test_downloadermiddleware.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 6168e0176..b538a0ed3 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -248,6 +248,12 @@ class MiddlewareUsingCoro(ManagerTestCase): @mark.only_asyncio() def test_asyncdef_asyncio(self): + if twisted_version < Version('twisted', 18, 4, 0): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'hangs when using Twisted versions lower than 18.4.0' + ) + resp = Response('http://example.com/index.html') class CoroMiddleware: From a3e53027ec35498dcb931404e02689877da6aeb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 14:16:26 +0100 Subject: [PATCH 376/568] Test HttpCompressionMiddleware subclasses with custom, parameterless __init__ --- .../downloadermiddlewares/httpcompression.py | 12 +++++-- ...st_downloadermiddleware_httpcompression.py | 36 ++++++++++++++++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 87f744956..1808154d2 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,9 +1,11 @@ +import warnings import zlib -from scrapy.utils.gz import gunzip +from scrapy.exceptions import NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.exceptions import NotConfigured +from scrapy.utils.deprecate import ScrapyDeprecationWarning +from scrapy.utils.gz import gunzip ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] @@ -28,6 +30,12 @@ class HttpCompressionMiddleware: try: return cls(stats=crawler.stats) except TypeError: + warnings.warn( + "HttpCompressionMiddleware subclasses must either modify " + "their '__init__' method to support a 'stats' parameter or " + "reimplement the 'from_crawler' method.", + ScrapyDeprecationWarning, + ) result = cls() result.stats = crawler.stats return result diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index c2bcbd63c..2ee87aa67 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,12 +1,13 @@ -from io import BytesIO -from unittest import TestCase, SkipTest -from os.path import join from gzip import GzipFile +from io import BytesIO +from os.path import join +from unittest import TestCase, SkipTest +from warnings import catch_warnings from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler @@ -321,3 +322,30 @@ class HttpCompressionTest(TestCase): self.assertEqual(response.body, b'') self.assertStatsEqual('httpcompression/response_count', None) self.assertStatsEqual('httpcompression/response_bytes', None) + + +class HttpCompressionSubclassTest(TestCase): + + def test_init_missing_stats(self): + class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware): + + def __init__(self): + super().__init__() + + crawler = get_crawler(Spider) + with catch_warnings(record=True) as caught_warnings: + instance = HttpCompressionMiddlewareSubclass.from_crawler(crawler) + messages = tuple( + str(warning.message) for warning in caught_warnings + if warning.category is ScrapyDeprecationWarning + ) + self.assertEqual( + messages, + ( + ( + "HttpCompressionMiddleware subclasses must either modify " + "their '__init__' method to support a 'stats' parameter " + "or reimplement the 'from_crawler' method." + ), + ) + ) From 3095d39740c4818d2c1c98d392255981ebed2a10 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 6 Nov 2020 12:16:10 -0300 Subject: [PATCH 377/568] Test: disable asyncio reactor on Windows for Py>=3.8 --- tests/test_crawler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0faaa79a3..ab113710d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -373,6 +373,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) From 1941f607ca54694d4822933eba973665baa1b45b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 16:25:56 +0100 Subject: [PATCH 378/568] Skip 2 additional tests with older Twisted versions --- tests/test_utils_signal.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index b66588efb..18a8fb49c 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,11 +1,14 @@ import asyncio +from unittest import SkipTest +from pydispatch import dispatcher from pytest import mark from testfixtures import LogCapture -from twisted.trial import unittest -from twisted.python.failure import Failure +from twisted import version as twisted_version from twisted.internet import defer, reactor -from pydispatch import dispatcher +from twisted.python.failure import Failure +from twisted.python.versions import Version +from twisted.trial import unittest from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.utils.test import get_from_asyncio_queue @@ -68,6 +71,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): return d +@mark.usefixtures('reactor_pytest') class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): async def ok_handler(self, arg, handlers_called): @@ -76,6 +80,19 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): await defer.succeed(42) return "OK" + def test_send_catch_log(self): + if ( + self.reactor_pytest == 'asyncio' + and twisted_version < Version('twisted', 18, 4, 0) + ): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'fails due to a timeout when using AsyncIO and Twisted ' + 'versions lower than 18.4.0' + ) + + return super().test_send_catch_log() + @mark.only_asyncio() class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): @@ -86,6 +103,16 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): await asyncio.sleep(0.2) return await get_from_asyncio_queue("OK") + def test_send_catch_log(self): + if (twisted_version < Version('twisted', 18, 4, 0): + raise SkipTest( + 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' + 'fails due to a timeout when using Twisted versions lower ' + 'than 18.4.0' + ) + + return super().test_send_catch_log() + class SendCatchLogTest2(unittest.TestCase): From ee98771fa72aa5e109292af5d734e5c49bae64e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 16:42:32 +0100 Subject: [PATCH 379/568] Remove unused variable --- tests/test_downloadermiddleware_httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 2ee87aa67..38d8534ca 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -334,7 +334,7 @@ class HttpCompressionSubclassTest(TestCase): crawler = get_crawler(Spider) with catch_warnings(record=True) as caught_warnings: - instance = HttpCompressionMiddlewareSubclass.from_crawler(crawler) + HttpCompressionMiddlewareSubclass.from_crawler(crawler) messages = tuple( str(warning.message) for warning in caught_warnings if warning.category is ScrapyDeprecationWarning From 4b28da433384f67d0e6c7d4d557f5bcf268ae846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 16:46:22 +0100 Subject: [PATCH 380/568] Fix syntax error --- tests/test_utils_signal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 18a8fb49c..ad7394232 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -104,7 +104,7 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): return await get_from_asyncio_queue("OK") def test_send_catch_log(self): - if (twisted_version < Version('twisted', 18, 4, 0): + if twisted_version < Version('twisted', 18, 4, 0): raise SkipTest( 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' 'fails due to a timeout when using Twisted versions lower ' From 114229eb4a5e3e0289000500cf063518be908d40 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 6 Nov 2020 13:29:14 -0300 Subject: [PATCH 381/568] Docs: add a note about asyncio.set_event_loop --- docs/topics/settings.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 912757850..0086a6c74 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -249,19 +249,25 @@ ASYNCIO_EVENT_LOOP Default: ``None`` -Import path of a given asyncio event loop class. +Import path of a given ``asyncio`` event loop class. -If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the -asyncio event loop to be used with it. Set the setting to the import path of the +If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the +asyncio event loop to be used with it. Set the setting to the import path of the desired asyncio event loop class. If the setting is set to ``None`` the default asyncio event loop will be used. If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor` -function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop -class to be used. +function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop +class to be used. Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`. +.. caution:: Please be aware that, when using a non-default event loop + (either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with + :func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call + :func:`asyncio.set_event_loop`, which will set the specified event loop + as the current loop for the current OS thread. + .. setting:: BOT_NAME BOT_NAME From a2c4a7f9200a06b227a297b9dd2919fe3ec37dbe Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Sun, 8 Nov 2020 19:12:18 -0300 Subject: [PATCH 382/568] Add missing f-string prefix to genspider output --- scrapy/commands/genspider.py | 2 +- tests/test_commands.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 72248bded..5f44daa70 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -98,7 +98,7 @@ class Command(ScrapyCommand): print(f"Created spider {name!r} using template {template_name!r} ", end=('' if spiders_module else '\n')) if spiders_module: - print("in module:\n {spiders_module.__name__}.{module}") + print(f"in module:\n {spiders_module.__name__}.{module}") def _find_template(self, template): template_file = join(self.templates_dir, f'{template}.tmpl') diff --git a/tests/test_commands.py b/tests/test_commands.py index 85aee55a5..d3ac05eac 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -389,8 +389,9 @@ class GenspiderCommandTest(CommandTest): def test_template(self, tplname='crawl'): args = [f'--template={tplname}'] if tplname else [] spname = 'test_spider' + spmodule = f"{self.project_name}.spiders.{spname}" p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module", out) + self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) p, out, err = self.proc('genspider', spname, 'test.com', *args) From 7e98a76ac455a8c69950104766719cde313bbb74 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 9 Nov 2020 12:17:15 -0300 Subject: [PATCH 383/568] Use deferred_from_coro in asyncio test --- tests/CrawlerProcess/asyncio_deferred_signal.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index bce300afe..dd82aa2ff 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -1,9 +1,9 @@ import asyncio import sys -import scrapy - +from scrapy import Spider from scrapy.crawler import CrawlerProcess +from scrapy.utils.defer import deferred_from_coro from twisted.internet.defer import Deferred @@ -14,13 +14,13 @@ class UppercasePipeline: def open_spider(self, spider): loop = asyncio.get_event_loop() - return Deferred.fromFuture(loop.create_task(self._open_spider(spider))) + return deferred_from_coro(self._open_spider(spider)) def process_item(self, item, spider): return {"url": item["url"].upper()} -class UrlSpider(scrapy.Spider): +class UrlSpider(Spider): name = "url_spider" start_urls = ["data:,"] custom_settings = { From b20cfef1e54d9f22769f6d0ec6ae06031bf86ec3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 9 Nov 2020 13:58:52 -0300 Subject: [PATCH 384/568] Remove unnecessary line from test --- tests/CrawlerProcess/asyncio_deferred_signal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index dd82aa2ff..46c2a12a4 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -13,7 +13,6 @@ class UppercasePipeline: await asyncio.sleep(0.1) def open_spider(self, spider): - loop = asyncio.get_event_loop() return deferred_from_coro(self._open_spider(spider)) def process_item(self, item, spider): From c20b34269f488dae4de9433d9c7c783bc481bc6f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 10 Nov 2020 15:35:09 -0300 Subject: [PATCH 385/568] Remove unnecessary pytest-azurepipelines package (#4876) --- tests/requirements-py3.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2eed2f5da..7f8a5c52e 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -6,7 +6,6 @@ mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 -pytest-azurepipelines pytest-cov pytest-twisted >= 1.11 pytest-xdist From 99cc853d6953d336ca65e0eecc0cb3286306bacf Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 29 Sep 2020 23:53:37 -0300 Subject: [PATCH 386/568] Response.protocol attribute --- scrapy/core/downloader/handlers/http11.py | 1 + scrapy/http/response/__init__.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1b041c8a8..0f30b01f9 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -442,6 +442,7 @@ class ScrapyAgent: flags=result["flags"], certificate=result["certificate"], ip_address=result["ip_address"], + protocol=getattr(result["txresponse"], "version", None), ) if result.get("failure"): result["failure"].value.response = response diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c635fde69..f4ef79c72 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -17,8 +17,18 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): - def __init__(self, url, status=200, headers=None, body=b'', flags=None, - request=None, certificate=None, ip_address=None): + def __init__( + self, + url, + status=200, + headers=None, + body=b"", + flags=None, + request=None, + certificate=None, + ip_address=None, + protocol=None, + ): self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) @@ -27,6 +37,7 @@ class Response(object_ref): self.flags = [] if flags is None else list(flags) self.certificate = certificate self.ip_address = ip_address + self.protocol = protocol @property def cb_kwargs(self): @@ -90,7 +101,7 @@ class Response(object_ref): given new values. """ for x in ['url', 'status', 'headers', 'body', - 'request', 'flags', 'certificate', 'ip_address']: + 'request', 'flags', 'certificate', 'ip_address', 'protocol']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) From 5b6b56240c24d02ef69e6cc591ffb2529bc3f36a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:08:37 -0300 Subject: [PATCH 387/568] Test Response.protocol attribute --- tests/test_downloader_handlers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 3e8d7e6b9..eb6d40df7 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -489,6 +489,13 @@ class Http11TestCase(HttpTestCase): def test_download_broken_chunked_content_allow_data_loss_via_setting(self): return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked') + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, (b"HTTP", 1, 1)) + return d + class Https11TestCase(Http11TestCase): scheme = 'https' From 587b4dd71fca12fa5fcc766b891540af77cb27c8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:20:50 -0300 Subject: [PATCH 388/568] Docs for the Response.protocol attribute --- docs/topics/request-response.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f3aaa2c8f..d7d5cd44e 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -693,9 +693,22 @@ Response objects :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` + :param protocol: A tuple containing information about the protocol that was used + to download the response. Taken from the ``version`` attribute of the + corresponding :class:`twisted.web.client.Response` object, it will tipically + consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` + to represent "HTTP/1.1". + :type protocol: :class:`tuple` + + .. versionadded:: 2.X.X + The ``protocol`` parameter. + .. versionadded:: 2.1.0 The ``ip_address`` parameter. + .. versionadded:: 2.0.0 + The ``certificate`` parameter. + .. attribute:: Response.url A string containing the URL of the response. @@ -780,6 +793,8 @@ Response objects .. attribute:: Response.certificate + .. versionadded:: 2.0.0 + A :class:`twisted.internet.ssl.Certificate` object representing the server's SSL certificate. @@ -795,6 +810,20 @@ Response objects handler, i.e. for ``http(s)`` responses. For other handlers, :attr:`ip_address` is always ``None``. + .. attribute:: Response.protocol + + .. versionadded:: 2.X.X + + A tuple containing information about the protocol that was used + to download the response. Taken from the ``version`` attribute of the + corresponding :class:`twisted.web.client.Response` object, it will tipically + consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` + to represent "HTTP/1.1". + + This attribute is currently only populated by the HTTP 1.1 download + handler, i.e. for ``http(s)`` responses. For other handlers, + :attr:`protocol` is always ``None``. + .. method:: Response.copy() Returns a new Response which is a copy of this Response. From 0fb7bcb2cf1606f63f1863bea254a34386c6a0f1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:26:03 -0300 Subject: [PATCH 389/568] Style adjustment --- scrapy/http/response/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f4ef79c72..185a9bb67 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -100,8 +100,9 @@ class Response(object_ref): """Create a new Response with the same attributes except for those given new values. """ - for x in ['url', 'status', 'headers', 'body', - 'request', 'flags', 'certificate', 'ip_address', 'protocol']: + for x in [ + "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol", + ]: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) From 61d089485c7ba66649b936e34833b2013fa12458 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:31:15 -0300 Subject: [PATCH 390/568] Docs: sort versionadded directives --- docs/topics/request-response.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d7d5cd44e..f1be41dde 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -700,14 +700,14 @@ Response objects to represent "HTTP/1.1". :type protocol: :class:`tuple` - .. versionadded:: 2.X.X - The ``protocol`` parameter. + .. versionadded:: 2.0.0 + The ``certificate`` parameter. .. versionadded:: 2.1.0 The ``ip_address`` parameter. - .. versionadded:: 2.0.0 - The ``certificate`` parameter. + .. versionadded:: 2.X.X + The ``protocol`` parameter. .. attribute:: Response.url From 22424125560496c9d131c9b7226aaf0f794e5ad8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 10:50:54 -0300 Subject: [PATCH 391/568] Docs: placeholder for versionadded directive --- docs/topics/request-response.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f1be41dde..1cb724227 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -706,7 +706,7 @@ Response objects .. versionadded:: 2.1.0 The ``ip_address`` parameter. - .. versionadded:: 2.X.X + .. versionadded:: VERSION The ``protocol`` parameter. .. attribute:: Response.url @@ -812,7 +812,7 @@ Response objects .. attribute:: Response.protocol - .. versionadded:: 2.X.X + .. versionadded:: VERSION A tuple containing information about the protocol that was used to download the response. Taken from the ``version`` attribute of the From 5e9a99e6a1e940864d9157252592f04658eaf851 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 11:15:29 -0300 Subject: [PATCH 392/568] Reponse.protocol as string --- docs/topics/request-response.rst | 20 +++++++------------- scrapy/core/downloader/handlers/http11.py | 7 ++++++- scrapy/core/downloader/webclient.py | 4 ++-- tests/test_downloader_handlers.py | 9 ++++++++- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1cb724227..98906992d 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -693,12 +693,9 @@ Response objects :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` - :param protocol: A tuple containing information about the protocol that was used - to download the response. Taken from the ``version`` attribute of the - corresponding :class:`twisted.web.client.Response` object, it will tipically - consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` - to represent "HTTP/1.1". - :type protocol: :class:`tuple` + :param protocol: The protocol that was used to download the response. + For instance: "HTTP/1.0", "HTTP/1.1" + :type protocol: :class:`str` .. versionadded:: 2.0.0 The ``certificate`` parameter. @@ -814,14 +811,11 @@ Response objects .. versionadded:: VERSION - A tuple containing information about the protocol that was used - to download the response. Taken from the ``version`` attribute of the - corresponding :class:`twisted.web.client.Response` object, it will tipically - consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` - to represent "HTTP/1.1". + The protocol that was used to download the response. + For instance: "HTTP/1.0", "HTTP/1.1" - This attribute is currently only populated by the HTTP 1.1 download - handler, i.e. for ``http(s)`` responses. For other handlers, + This attribute is currently only populated by the HTTP download + handlers, i.e. for ``http(s)`` responses. For other handlers, :attr:`protocol` is always ``None``. .. method:: Response.copy() diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 0f30b01f9..c7553eb87 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -434,6 +434,11 @@ class ScrapyAgent: def _cb_bodydone(self, result, request, url): headers = Headers(result["txresponse"].headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) + try: + version = result["txresponse"].version + protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}" + except (AttributeError, TypeError): + protocol = None response = respcls( url=url, status=int(result["txresponse"].code), @@ -442,7 +447,7 @@ class ScrapyAgent: flags=result["flags"], certificate=result["certificate"], ip_address=result["ip_address"], - protocol=getattr(result["txresponse"], "version", None), + protocol=protocol, ) if result.get("failure"): result["failure"].value.response = response diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index c13683393..9524cce2b 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -7,7 +7,7 @@ from twisted.internet.protocol import ClientFactory from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, to_unicode from scrapy.responsetypes import responsetypes @@ -110,7 +110,7 @@ class ScrapyHTTPClientFactory(ClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) - return respcls(url=self._url, status=status, headers=headers, body=body) + return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version)) def _set_connection_attributes(self, request): parsed = urlparse_cached(request) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index eb6d40df7..a8763a7a5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -360,6 +360,13 @@ class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" download_handler_cls = HTTP10DownloadHandler + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.0") + return d + class Https10TestCase(Http10TestCase): scheme = 'https' @@ -493,7 +500,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL("host"), method="GET") d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, (b"HTTP", 1, 1)) + d.addCallback(self.assertEqual, "HTTP/1.1") return d From b0368228d7f6391c0df41fca1609c6548613ad6b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 11:18:03 -0300 Subject: [PATCH 393/568] Add exception to catch --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c7553eb87..a0fd837b1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -437,7 +437,7 @@ class ScrapyAgent: try: version = result["txresponse"].version protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}" - except (AttributeError, TypeError): + except (AttributeError, TypeError, IndexError): protocol = None response = respcls( url=url, From 034d61e6cb8391f045eb9b20824964e951549e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 11 Nov 2020 16:46:03 +0100 Subject: [PATCH 394/568] =?UTF-8?q?Restrict=20pip=E2=80=99s=20--use-featur?= =?UTF-8?q?e=3D2020-resolver=20to=20the=20extra-deps=20environment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 6149a9c58..f35466d05 100644 --- a/tox.ini +++ b/tox.ini @@ -29,8 +29,6 @@ passenv = download = true commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} -install_command = - pip install --use-feature=2020-resolver {opts} {packages} [testenv:typing] basepython = python3 @@ -111,6 +109,8 @@ deps = {[testenv]deps} reppy robotexclusionrulesparser +install_command = + pip install --use-feature=2020-resolver {opts} {packages} [testenv:asyncio] commands = From 85604e1078b927c2a875040e29c58b4594c8d386 Mon Sep 17 00:00:00 2001 From: joaquin garmendia Date: Wed, 11 Nov 2020 15:16:01 -0500 Subject: [PATCH 395/568] Add failed and success count stats to feedstorage backends (#4850) --- scrapy/extensions/feedexport.py | 22 ++++--- tests/test_feedexport.py | 106 ++++++++++++++++++++++++++++---- 2 files changed, 110 insertions(+), 18 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3fb4d0e2c..bec114707 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -319,18 +319,26 @@ class FeedExporter: # Use `largs=log_args` to copy log_args into function's scope # instead of using `log_args` from the outer scope d.addCallback( - lambda _, largs=log_args: logger.info( - logfmt % "Stored", largs, extra={'spider': spider} - ) + self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__ ) d.addErrback( - lambda f, largs=log_args: logger.error( - logfmt % "Error storing", largs, - exc_info=failure_to_exc_info(f), extra={'spider': spider} - ) + self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__ ) return d + def _handle_store_error(self, f, largs, logfmt, spider, slot_type): + logger.error( + logfmt % "Error storing", largs, + exc_info=failure_to_exc_info(f), extra={'spider': spider} + ) + self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") + + def _handle_store_success(self, f, largs, logfmt, spider, slot_type): + logger.info( + logfmt % "Stored", largs, extra={'spider': spider} + ) + self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") + def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): """ Redirect the output data stream to a new file. diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1ea4e8e12..d248824fc 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -8,6 +8,7 @@ import tempfile import warnings from abc import ABC, abstractmethod from collections import defaultdict +from contextlib import ExitStack from io import BytesIO from logging import getLogger from pathlib import Path @@ -47,6 +48,21 @@ from scrapy.utils.test import ( ) from tests.mockserver import MockFTPServer, MockServer +from tests.spiders import ItemSpider + + +def path_to_url(path): + return urljoin('file:', pathname2url(str(path))) + + +def printf_escape(string): + return string.replace('%', '%%') + + +def build_url(path): + if path[0] != '/': + path = '/' + path + return urljoin('file:', path) class FileFeedStorageTest(unittest.TestCase): @@ -620,12 +636,6 @@ class FeedExportTest(FeedExportTestBase): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ - def path_to_url(path): - return urljoin('file:', pathname2url(str(path))) - - def printf_escape(string): - return string.replace('%', '%%') - FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { printf_escape(path_to_url(file_path)): feed_options @@ -748,6 +758,69 @@ class FeedExportTest(FeedExportTestBase): result = self._load_until_eof(data['marshal'], load_func=marshal.load) self.assertEqual(expected, result) + @defer.inlineCallbacks + def test_stats_file_success(self): + settings = { + "FEEDS": { + printf_escape(path_to_url(self._random_temp_filename())): { + "format": "json", + } + }, + } + crawler = get_crawler(ItemSpider, settings) + with MockServer() as mockserver: + yield crawler.crawl(mockserver=mockserver) + self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1) + + @defer.inlineCallbacks + def test_stats_file_failed(self): + settings = { + "FEEDS": { + printf_escape(path_to_url(self._random_temp_filename())): { + "format": "json", + } + }, + } + crawler = get_crawler(ItemSpider, settings) + with ExitStack() as stack: + mockserver = stack.enter_context(MockServer()) + stack.enter_context( + mock.patch( + "scrapy.extensions.feedexport.FileFeedStorage.store", + side_effect=KeyError("foo")) + ) + yield crawler.crawl(mockserver=mockserver) + self.assertIn("feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats()) + self.assertEqual(crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1) + + @defer.inlineCallbacks + def test_stats_multiple_file(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + "FEEDS": { + printf_escape(path_to_url(self._random_temp_filename())): { + "format": "json", + }, + "s3://bucket/key/foo.csv": { + "format": "csv", + }, + "stdout:": { + "format": "xml", + } + }, + } + crawler = get_crawler(ItemSpider, settings) + with MockServer() as mockserver, mock.patch.object(S3FeedStorage, "store"): + yield crawler.crawl(mockserver=mockserver) + self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()) + self.assertIn("feedexport/success_count/S3FeedStorage", crawler.stats.get_stats()) + self.assertIn("feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats()) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1) + @defer.inlineCallbacks def test_export_items(self): # feed exporters use field names from Item @@ -1256,11 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ - def build_url(path): - if path[0] != '/': - path = '/' + path - return urljoin('file:', path) - FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { build_url(file_path): feed @@ -1550,6 +1618,22 @@ class BatchDeliveriesTest(FeedExportTestBase): data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) + @defer.inlineCallbacks + def test_stats_batch_file_success(self): + settings = { + "FEEDS": { + build_url(os.path.join(self._random_temp_filename(), "json", self._file_mark)): { + "format": "json", + } + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + crawler = get_crawler(ItemSpider, settings) + with MockServer() as mockserver: + yield crawler.crawl(total=2, mockserver=mockserver) + self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12) + @defer.inlineCallbacks def test_s3_export(self): skip_if_no_boto() From 2405df49f14cbc052d73e58a819a87417b2502e8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 16 Nov 2020 12:50:33 -0300 Subject: [PATCH 396/568] Add tests for Response.protocol=None --- tests/test_downloader_handlers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a8763a7a5..f51a6cd3c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -115,6 +115,7 @@ class FileTestCase(unittest.TestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.status, 200) self.assertEqual(response.body, b'0123456789') + self.assertEqual(response.protocol, None) request = Request(path_to_file_uri(self.tmpname + '^')) assert request.url.upper().endswith('%5E') @@ -976,6 +977,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'I have the power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) + self.assertIsNone(r.protocol) return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): @@ -1134,3 +1136,10 @@ class DataURITestCase(unittest.TestCase): request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') return self.download_request(request, self.spider).addCallback(_test) + + def test_protocol(self): + def _test(response): + self.assertIsNone(response.protocol) + + request = Request("data:,") + return self.download_request(request, self.spider).addCallback(_test) From 15d301e968aa3e26a28f771cf1b45635e84ef094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Nov 2020 09:16:08 +0100 Subject: [PATCH 397/568] Cover Scrapy 2.4.1 in the release notes (#4884) Co-authored-by: Mikhail Korobov --- docs/news.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index a3889705d..e92493252 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,29 @@ Release notes ============= +.. _release-2.4.1: + +Scrapy 2.4.1 (2020-11-17) +------------------------- + +- Fixed :ref:`feed exports ` overwrite support (:issue:`4845`, :issue:`4857`, :issue:`4859`) + +- Fixed the AsyncIO event loop handling, which could make code hang + (:issue:`4855`, :issue:`4872`) + +- Fixed the IPv6-capable DNS resolver + :class:`~scrapy.resolver.CachingHostnameResolver` for download handlers + that call + :meth:`reactor.resolve ` + (:issue:`4802`, :issue:`4803`) + +- Fixed the output of the :command:`genspider` command showing placeholders + instead of the import part of the generated spider module (:issue:`4874`) + +- Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`, + :issue:`4876`) + + .. _release-2.4.0: Scrapy 2.4.0 (2020-10-11) From 26836c4e1ae9588ee173c5977fc6611364ca7cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Nov 2020 09:17:39 +0100 Subject: [PATCH 398/568] =?UTF-8?q?Bump=20version:=202.4.0=20=E2=86=92=202?= =?UTF-8?q?.4.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0f142472e..956c512cb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.0 +current_version = 2.4.1 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 197c4d5c2..005119baa 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.4.0 +2.4.1 From 63becd1bc89395750b39139e2114193607f3ca61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Nov 2020 21:58:08 +0100 Subject: [PATCH 399/568] Update news.rst --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index e92493252..0391506c4 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -20,7 +20,7 @@ Scrapy 2.4.1 (2020-11-17) (:issue:`4802`, :issue:`4803`) - Fixed the output of the :command:`genspider` command showing placeholders - instead of the import part of the generated spider module (:issue:`4874`) + instead of the import path of the generated spider module (:issue:`4874`) - Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`, :issue:`4876`) From 6ef3dc2029fea1f692aa3054ea47d799772151d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Nov 2020 22:28:20 +0100 Subject: [PATCH 400/568] Use the new pip resolver for Tox environments with pinned dependencies --- tox.ini | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tox.ini b/tox.ini index f35466d05..56e736fbf 100644 --- a/tox.ini +++ b/tox.ini @@ -88,12 +88,18 @@ deps = botocore==1.4.87 google-cloud-storage==1.29.0 Pillow==4.0.0 +install_command = + # --use-feature=2020-resolver is required, otherwise the latest verion of + # Twisted gets installed. + pip install --use-feature=2020-resolver {opts} {packages} [testenv:pinned] deps = {[pinned]deps} lxml==3.5.0 PyDispatcher==2.0.5 +install_command = + {[pinned]install_command} [testenv:windows-pinned] basepython = python3 @@ -103,6 +109,8 @@ deps = # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 PyDispatcher==2.0.5 +install_command = + {[pinned]install_command} [testenv:extra-deps] deps = @@ -110,6 +118,8 @@ deps = reppy robotexclusionrulesparser install_command = + # Test --use-feature=2020-resolver for the latest version of all + # dependencies. pip install --use-feature=2020-resolver {opts} {packages} [testenv:asyncio] @@ -118,6 +128,8 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} +install_command = + {[pinned]install_command} commands = {[testenv:asyncio]commands} [testenv:pypy3] @@ -131,6 +143,8 @@ deps = {[pinned]deps} lxml==4.0.0 PyPyDispatcher==2.1.0 +install_command = + {[pinned]install_command} commands = {[testenv:pypy3]commands} [docs] From bde96a5ad98caed74cbaaeaf1dfe4b215093c03f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Nov 2020 16:42:44 +0100 Subject: [PATCH 401/568] Ignore server-initiated events --- scrapy/core/http2/protocol.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 9d499596c..67a86bd62 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -349,10 +349,16 @@ class H2ClientProtocol(Protocol, TimeoutMixin): ]) def data_received(self, event: DataReceived) -> None: - self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) + try: + self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) + except KeyError: + logger.debug(f'Ignoring server-initiated event {event}') def response_received(self, event: ResponseReceived) -> None: - self.streams[event.stream_id].receive_headers(event.headers) + try: + self.streams[event.stream_id].receive_headers(event.headers) + except KeyError: + logger.debug(f'Ignoring server-initiated event {event}') def settings_acknowledged(self, event: SettingsAcknowledged) -> None: self.metadata['settings_acknowledged'] = True @@ -365,12 +371,20 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) def stream_ended(self, event: StreamEnded) -> None: - stream = self.pop_stream(event.stream_id) - stream.close(StreamCloseReason.ENDED, from_protocol=True) + try: + stream = self.pop_stream(event.stream_id) + except KeyError: + logger.debug(f'Ignoring server-initiated event {event}') + else: + stream.close(StreamCloseReason.ENDED, from_protocol=True) def stream_reset(self, event: StreamReset) -> None: - stream = self.pop_stream(event.stream_id) - stream.close(StreamCloseReason.RESET, from_protocol=True) + try: + stream = self.pop_stream(event.stream_id) + except KeyError: + logger.debug(f'Ignoring server-initiated event {event}') + else: + stream.close(StreamCloseReason.RESET, from_protocol=True) def window_updated(self, event: WindowUpdated) -> None: if event.stream_id != 0: From 08f5ed712f4fdafec122f887c3ef06629f35ee0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Nov 2020 17:38:18 +0100 Subject: [PATCH 402/568] Fix memory issue due to unexpectedly large server frames --- scrapy/core/http2/protocol.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 67a86bd62..d8d0974b8 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -13,7 +13,7 @@ from h2.events import ( SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived, WindowUpdated ) -from h2.exceptions import H2Error +from h2.exceptions import FrameTooLargeError, H2Error from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory @@ -261,6 +261,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin): events = self.conn.receive_data(data) self._handle_events(events) except H2Error as e: + if isinstance(e, FrameTooLargeError): + # hyper-h2 does not drop the connection in this scenario, we + # need to abort the connection manually. + self._conn_lost_errors += [e] + self.transport.abortConnection() + return + # Save this error as ultimately the connection will be dropped # internally by hyper-h2. Saved error will be passed to all the streams # closed with the connection. From 075ab156b0ac7bf56828fd40a843f3e9b63a3de3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 11:59:59 -0300 Subject: [PATCH 403/568] Deprecate scrapy.utils.py36 module --- scrapy/utils/py36.py | 17 +++++++++-------- scrapy/utils/python.py | 7 +++++++ scrapy/utils/spider.py | 7 ++----- setup.cfg | 3 --- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py index c8c24076e..070b145a4 100644 --- a/scrapy/utils/py36.py +++ b/scrapy/utils/py36.py @@ -1,10 +1,11 @@ -""" -Helpers using Python 3.6+ syntax (ignore SyntaxError on import). -""" +import warnings + +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.python import collect_asyncgen # noqa: F401 -async def collect_asyncgen(result): - results = [] - async for x in result: - results.append(x) - return results +warnings.warn( + "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.python` instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, +) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5703fd4c3..7bd286523 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -355,3 +355,10 @@ class MutableChain: @deprecated("scrapy.utils.python.MutableChain.__next__") def next(self): return self.__next__() + + +async def collect_asyncgen(result): + results = [] + async for x in result: + results.append(x) + return results diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index f3a9a67a3..5319604ba 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,17 +4,14 @@ import logging from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter -try: - from scrapy.utils.py36 import collect_asyncgen -except SyntaxError: - collect_asyncgen = None +from scrapy.utils.python import collect_asyncgen logger = logging.getLogger(__name__) def iterate_spider_output(result): - if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result): + if inspect.isasyncgen(result): d = deferred_from_coro(collect_asyncgen(result)) d.addCallback(iterate_spider_output) return d diff --git a/setup.cfg b/setup.cfg index 8101443e3..0c9f6b963 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,9 +55,6 @@ ignore_errors = True [mypy-scrapy.utils.response] ignore_errors = True -[mypy-scrapy.utils.spider] -ignore_errors = True - [mypy-scrapy.utils.trackref] ignore_errors = True From 4075e1eadd0a9e83356d18b9ca73090e4a0bc770 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 15:07:56 -0300 Subject: [PATCH 404/568] Remove deprecated modules (utils.http/markup/multipart) --- pytest.ini | 3 --- scrapy/utils/http.py | 36 ------------------------------------ scrapy/utils/markup.py | 14 -------------- scrapy/utils/multipart.py | 15 --------------- 4 files changed, 68 deletions(-) delete mode 100644 scrapy/utils/http.py delete mode 100644 scrapy/utils/markup.py delete mode 100644 scrapy/utils/multipart.py diff --git a/pytest.ini b/pytest.ini index 1c95f715a..d4deeb57c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -36,8 +36,5 @@ flake8-ignore = scrapy/spiders/__init__.py E402 F401 # Issues pending a review: - scrapy/utils/http.py F403 - scrapy/utils/markup.py F403 - scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 tests/test_loader.py E741 diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py deleted file mode 100644 index ceb3f0509..000000000 --- a/scrapy/utils/http.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.http instead of this module -""" - -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.decorators import deprecated -from w3lib.http import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.http` is deprecated, " - "Please import from `w3lib.http` instead.", - ScrapyDeprecationWarning, stacklevel=2) - - -@deprecated -def decode_chunked_transfer(chunked_body): - """Parsed body received with chunked transfer encoding, and return the - decoded body. - - For more info see: - https://en.wikipedia.org/wiki/Chunked_transfer_encoding - - """ - body, h, t = '', '', chunked_body - while t: - h, t = t.split('\r\n', 1) - if h == '0': - break - size = int(h, 16) - body += t[:size] - t = t[size + 2:] - return body diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py deleted file mode 100644 index 9728c542a..000000000 --- a/scrapy/utils/markup.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.html instead of this module -""" -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.html import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.markup` is deprecated. " - "Please import from `w3lib.html` instead.", - ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py deleted file mode 100644 index 5dcf791b8..000000000 --- a/scrapy/utils/multipart.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.form instead of this module -""" -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.form import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.multipart` is deprecated. " - "If you're using `encode_multipart` function, please use " - "`urllib3.filepost.encode_multipart_formdata` instead", - ScrapyDeprecationWarning, stacklevel=2) From 51ca4d0138e5c9cf637074f59c839ff9b5839db6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 15:47:08 -0300 Subject: [PATCH 405/568] Remove deprecated scrapy.utils.gz.is_gzipped function --- scrapy/utils/gz.py | 15 +----------- tests/test_utils_gz.py | 55 ++++++++++++------------------------------ 2 files changed, 16 insertions(+), 54 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 11d433cf5..76156a4b8 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,7 +1,6 @@ +import struct from gzip import GzipFile from io import BytesIO -import re -import struct from scrapy.utils.decorators import deprecated @@ -42,17 +41,5 @@ def gunzip(data): return b''.join(output_list) -_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search -_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search - - -@deprecated -def is_gzipped(response): - """Return True if the response is gzipped, or False otherwise""" - ctype = response.headers.get('Content-Type', b'') - cenc = response.headers.get('Content-Encoding', b'').lower() - return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip') - - def gzip_magic_number(response): return response.body[:3] == b'\x1f\x8b\x08' diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 7148185f4..4943731cb 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -3,10 +3,11 @@ from os.path import join from w3lib.encoding import html_to_unicode -from scrapy.utils.gz import gunzip, is_gzipped -from scrapy.http import Response, Headers +from scrapy.utils.gz import gunzip, gzip_magic_number +from scrapy.http import Response from tests import tests_datadir + SAMPLEDIR = join(tests_datadir, 'compressed') @@ -14,8 +15,12 @@ class GunzipTest(unittest.TestCase): def test_gunzip_basic(self): with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f: - text = gunzip(f.read()) - self.assertEqual(len(text), 9950) + r1 = Response("http://www.example.com", body=f.read()) + self.assertTrue(gzip_magic_number(r1)) + + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + self.assertFalse(gzip_magic_number(r2)) + self.assertEqual(len(r2.body), 9950) def test_gunzip_truncated(self): with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f: @@ -28,46 +33,16 @@ class GunzipTest(unittest.TestCase): def test_gunzip_truncated_short(self): with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f: - text = gunzip(f.read()) - assert text.endswith(b'') + r1 = Response("http://www.example.com", body=f.read()) + self.assertTrue(gzip_magic_number(r1)) - def test_is_x_gzipped_right(self): - hdrs = Headers({"Content-Type": "application/x-gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - def test_is_gzipped_right(self): - hdrs = Headers({"Content-Type": "application/gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - def test_is_gzipped_not_quite(self): - hdrs = Headers({"Content-Type": "application/gzippppp"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_case_insensitive(self): - hdrs = Headers({"Content-Type": "Application/X-Gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - hdrs = Headers({"Content-Type": "application/X-GZIP ; charset=utf-8"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert r2.body.endswith(b'') + self.assertFalse(gzip_magic_number(r2)) def test_is_gzipped_empty(self): r1 = Response("http://www.example.com") - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_wrong(self): - hdrs = Headers({"Content-Type": "application/javascript"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_with_charset(self): - hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) + self.assertFalse(gzip_magic_number(r1)) def test_gunzip_illegal_eof(self): with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f: From 462014bc5738a5ed18ec4c15a91384eb17e57096 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 15:51:59 -0300 Subject: [PATCH 406/568] Scheduler: remove support for deprecated queuelib.PriorityQueue --- scrapy/core/scheduler.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index a18c26b17..9ce823dbc 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,14 +1,10 @@ import os import json import logging -import warnings from os.path import join, exists -from queuelib import PriorityQueue - from scrapy.utils.misc import load_object, create_instance from scrapy.utils.job import job_dir -from scrapy.utils.deprecate import ScrapyDeprecationWarning logger = logging.getLogger(__name__) @@ -56,14 +52,6 @@ class Scheduler: dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) dupefilter = create_instance(dupefilter_cls, settings, crawler) pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) - if pqclass is PriorityQueue: - warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'" - " is no longer supported because of API changes; " - "please use 'scrapy.pqueues.ScrapyPriorityQueue'", - ScrapyDeprecationWarning) - from scrapy.pqueues import ScrapyPriorityQueue - pqclass = ScrapyPriorityQueue - dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) logunser = settings.getbool('SCHEDULER_DEBUG') From 0a93df9efd20a4bcd34d2ee5e6bdf6d365d70409 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 16:16:18 -0300 Subject: [PATCH 407/568] Move collect_asyncgen to utils.asyncgen --- scrapy/utils/asyncgen.py | 5 +++++ scrapy/utils/py36.py | 4 ++-- scrapy/utils/python.py | 7 ------- 3 files changed, 7 insertions(+), 9 deletions(-) create mode 100644 scrapy/utils/asyncgen.py diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py new file mode 100644 index 000000000..7f697af5f --- /dev/null +++ b/scrapy/utils/asyncgen.py @@ -0,0 +1,5 @@ +async def collect_asyncgen(result): + results = [] + async for x in result: + results.append(x) + return results diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py index 070b145a4..653e2bbbb 100644 --- a/scrapy/utils/py36.py +++ b/scrapy/utils/py36.py @@ -1,11 +1,11 @@ import warnings from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.python import collect_asyncgen # noqa: F401 +from scrapy.utils.asyncgen import collect_asyncgen # noqa: F401 warnings.warn( - "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.python` instead.", + "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.", category=ScrapyDeprecationWarning, stacklevel=2, ) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7bd286523..5703fd4c3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -355,10 +355,3 @@ class MutableChain: @deprecated("scrapy.utils.python.MutableChain.__next__") def next(self): return self.__next__() - - -async def collect_asyncgen(result): - results = [] - async for x in result: - results.append(x) - return results From 18b05af87783d71f9c8f9f1ebe027083037e6f86 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 16:18:58 -0300 Subject: [PATCH 408/568] Remove tests/test_utils_http.py --- tests/test_utils_http.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 tests/test_utils_http.py diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py deleted file mode 100644 index 363b015a8..000000000 --- a/tests/test_utils_http.py +++ /dev/null @@ -1,19 +0,0 @@ -import unittest - -from scrapy.utils.http import decode_chunked_transfer - - -class ChunkedTest(unittest.TestCase): - - def test_decode_chunked_transfer(self): - """Example taken from: http://en.wikipedia.org/wiki/Chunked_transfer_encoding""" - chunked_body = "25\r\n" + "This is the data in the first chunk\r\n\r\n" - chunked_body += "1C\r\n" + "and this is the second one\r\n\r\n" - chunked_body += "3\r\n" + "con\r\n" - chunked_body += "8\r\n" + "sequence\r\n" - chunked_body += "0\r\n\r\n" - body = decode_chunked_transfer(chunked_body) - self.assertEqual( - body, - "This is the data in the first chunk\r\nand this is the second one\r\nconsequence" - ) From fe8bb6bd905ad2a733ba5e876f4197888605902e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 16:51:04 -0300 Subject: [PATCH 409/568] Fix import --- scrapy/utils/spider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 5319604ba..59fc9202f 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,7 +4,7 @@ import logging from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import collect_asyncgen +from scrapy.utils.asyncgen import collect_asyncgen logger = logging.getLogger(__name__) From 0dc3e6350c230028addd8e73833c93341bab4b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 23 Nov 2020 22:10:45 +0100 Subject: [PATCH 410/568] Add a test to check the Twisted version in pinned environments --- tests/test_dependencies.py | 32 ++++++++++++++++++++++++++++++++ tox.ini | 10 ++++++++++ 2 files changed, 42 insertions(+) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 5d0a1d0c9..4e4f190ab 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,8 +1,14 @@ +import os +import re +from configparser import ConfigParser from importlib import import_module + +from twisted import version as twisted_version from twisted.trial import unittest class ScrapyUtilsTest(unittest.TestCase): + def test_required_openssl_version(self): try: module = import_module('OpenSSL') @@ -13,6 +19,32 @@ class ScrapyUtilsTest(unittest.TestCase): installed_version = [int(x) for x in module.__version__.split('.')[:2]] assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" + def test_pinned_twisted_version(self): + """When running tests within a Tox environment with pinned + dependencies, make sure that the version of Twisted is the pinned + version. + + See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 + """ + if not os.environ.get('SCRAPY_PINNED', None): + self.skipTest('Not in a pinned environment') + + tox_config_file_path = os.path.join( + os.path.dirname(__file__), + '..', + 'tox.ini', + ) + config_parser = ConfigParser() + config_parser.read(tox_config_file_path) + pattern = r'Twisted==([\d.]+)' + match = re.search(pattern, config_parser['pinned']['deps']) + pinned_twisted_version_string = match[1] + + self.assertEqual( + twisted_version.short(), + pinned_twisted_version_string + ) + if __name__ == "__main__": unittest.main() diff --git a/tox.ini b/tox.ini index 56e736fbf..ea71a2476 100644 --- a/tox.ini +++ b/tox.ini @@ -92,6 +92,8 @@ install_command = # --use-feature=2020-resolver is required, otherwise the latest verion of # Twisted gets installed. pip install --use-feature=2020-resolver {opts} {packages} +setenv = + SCRAPY_PINNED=true [testenv:pinned] deps = @@ -100,6 +102,8 @@ deps = PyDispatcher==2.0.5 install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:windows-pinned] basepython = python3 @@ -111,6 +115,8 @@ deps = PyDispatcher==2.0.5 install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:extra-deps] deps = @@ -131,6 +137,8 @@ deps = {[testenv:pinned]deps} install_command = {[pinned]install_command} commands = {[testenv:asyncio]commands} +setenv = + {[pinned]setenv} [testenv:pypy3] basepython = pypy3 @@ -146,6 +154,8 @@ deps = install_command = {[pinned]install_command} commands = {[testenv:pypy3]commands} +setenv = + {[pinned]setenv} [docs] changedir = docs From a752fa072e1acbb233191dbf04728db1fd6712a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 23 Nov 2020 22:58:54 +0100 Subject: [PATCH 411/568] Implement retry request functions and mixin --- scrapy/downloadermiddlewares/retry.py | 134 ++++++++++++++++++++------ 1 file changed, 106 insertions(+), 28 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 51fe59254..023ab7d60 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -31,6 +31,105 @@ from scrapy.utils.python import global_object_name logger = logging.getLogger(__name__) +def get_retry_request( + request, + *, + reason, + spider, + max_retry_times=None, + priority_adjust=None, +): + settings = spider.crawler.settings + stats = spider.crawler.stats + retry_times = request.meta.get('retry_times', 0) + 1 + request_max_retry_times = request.meta.get( + 'max_retry_times', + max_retry_times, + ) + if request_max_retry_times is None: + request_max_retry_times = settings.getint('RETRY_TIMES') + if retry_times <= request_max_retry_times: + logger.debug( + "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider} + ) + new_request = request.copy() + new_request.meta['retry_times'] = retry_times + new_request.dont_filter = True + if priority_adjust is None: + priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') + new_request.priority = request.priority + priority_adjust + + if isinstance(reason, Exception): + reason = global_object_name(reason.__class__) + + stats.inc_value('retry/count') + stats.inc_value(f'retry/reason_count/{reason}') + return new_request + else: + stats.inc_value('retry/max_reached') + logger.error("Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider}) + return None + + +def retry_request( + request, + *, + reason, + spider, + max_retry_times=None, + priority_adjust=None, +): + new_request = get_retry_request( + request, + reason=reason, + spider=spider, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, + ) + if new_request: + return [new_request] + return [] + + +class RetrySpiderMixin: + + def get_retry_request( + self, + request, + *, + reason, + max_retry_times=None, + priority_adjust=None, + ): + return get_retry_request( + request, + reason=reason, + spider=self, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, + ) + + def retry_request( + self, + request, + *, + reason, + max_retry_times=None, + priority_adjust=None, + ): + return retry_request( + request, + reason=reason, + spider=self, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, + ) + + class RetryMiddleware: # IOError is raised by the HttpCompression middleware when trying to @@ -67,31 +166,10 @@ class RetryMiddleware: return self._retry(request, exception, spider) def _retry(self, request, reason, spider): - retries = request.meta.get('retry_times', 0) + 1 - - retry_times = self.max_retry_times - - if 'max_retry_times' in request.meta: - retry_times = request.meta['max_retry_times'] - - stats = spider.crawler.stats - if retries <= retry_times: - logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s", - {'request': request, 'retries': retries, 'reason': reason}, - extra={'spider': spider}) - retryreq = request.copy() - retryreq.meta['retry_times'] = retries - retryreq.dont_filter = True - retryreq.priority = request.priority + self.priority_adjust - - if isinstance(reason, Exception): - reason = global_object_name(reason.__class__) - - stats.inc_value('retry/count') - stats.inc_value(f'retry/reason_count/{reason}') - return retryreq - else: - stats.inc_value('retry/max_reached') - logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s", - {'request': request, 'retries': retries, 'reason': reason}, - extra={'spider': spider}) + return get_retry_request( + request, + reason=reason, + spider=spider, + max_retry_times=self.max_retry_times, + priority_adjust=self.priority_adjust, + ) From f6879c681ebf94b22ee777f3249dfe59a99559cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 23 Nov 2020 23:53:03 +0100 Subject: [PATCH 412/568] =?UTF-8?q?SCRAPY=5FPINNED=20=E2=86=92=20=5FSCRAPY?= =?UTF-8?q?=5FPINNED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_dependencies.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 4e4f190ab..93e7311d2 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -26,7 +26,7 @@ class ScrapyUtilsTest(unittest.TestCase): See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 """ - if not os.environ.get('SCRAPY_PINNED', None): + if not os.environ.get('_SCRAPY_PINNED', None): self.skipTest('Not in a pinned environment') tox_config_file_path = os.path.join( diff --git a/tox.ini b/tox.ini index ea71a2476..66866301c 100644 --- a/tox.ini +++ b/tox.ini @@ -93,7 +93,7 @@ install_command = # Twisted gets installed. pip install --use-feature=2020-resolver {opts} {packages} setenv = - SCRAPY_PINNED=true + _SCRAPY_PINNED=true [testenv:pinned] deps = From 95d39d5cb464ca22516a30f96c0a323613421090 Mon Sep 17 00:00:00 2001 From: etimoz Date: Sun, 29 Nov 2020 13:24:04 +0100 Subject: [PATCH 413/568] removed wrong super argument in overriding serialize_fields code example --- docs/topics/exporters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index ef50c9f5c..0a0a1765a 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -123,7 +123,7 @@ Example:: def serialize_field(self, field, name, value): if field == 'price': return f'$ {str(value)}' - return super(Product, self).serialize_field(field, name, value) + return super().serialize_field(field, name, value) .. _topics-exporters-reference: From 7fec9f991f0bd415900df2bf288c6cda909ecd77 Mon Sep 17 00:00:00 2001 From: Kader DJEHAF Date: Mon, 30 Nov 2020 21:47:28 +0100 Subject: [PATCH 414/568] [Cleaned] PEP 8: E251 unexpected spaces around keyword / parameter equals (#4911) [Cleaned] PEP 8: E251 unexpected spaces around keyword / parameter equals --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0c2281400..35736b75f 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( name='Scrapy', version=version, url='https://scrapy.org', - project_urls = { + project_urls={ 'Documentation': 'https://docs.scrapy.org/', 'Source': 'https://github.com/scrapy/scrapy', 'Tracker': 'https://github.com/scrapy/scrapy/issues', From a80bafe5cdcb4b5e2542524fe641570f9107a121 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 30 Nov 2020 19:03:13 -0300 Subject: [PATCH 415/568] Remove deprecated SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE --- scrapy/utils/project.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index fd13d85e3..35e59a258 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -68,18 +68,10 @@ def get_project_settings(): if settings_module_path: settings.setmodule(settings_module_path, priority='project') - pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") - if pickled_settings: - warnings.warn("Use of environment variable " - "'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE' " - "is deprecated.", ScrapyDeprecationWarning) - settings.setdict(pickle.loads(pickled_settings), priority='project') - scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if k.startswith('SCRAPY_')} valid_envvars = { 'CHECK', - 'PICKLED_SETTINGS_TO_OVERRIDE', 'PROJECT', 'PYTHON_SHELL', 'SETTINGS_MODULE', From 6091f3cc03835b24def08bf358d7b19207be685d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Dec 2020 10:26:21 -0300 Subject: [PATCH 416/568] Remove unused pickle import --- scrapy/utils/project.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 35e59a258..c66af497e 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,5 +1,4 @@ import os -import pickle import warnings from importlib import import_module From ef09e0d10fc950ac308a159988be6b50e87bd906 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 19 Nov 2020 10:35:49 -0300 Subject: [PATCH 417/568] Some type hints --- scrapy/__init__.py | 2 +- scrapy/commands/__init__.py | 4 +- scrapy/commands/parse.py | 6 +- scrapy/contracts/__init__.py | 127 ++++++++++++++-------------- scrapy/http/cookies.py | 8 +- scrapy/item.py | 3 +- scrapy/mail.py | 7 +- scrapy/spidermiddlewares/referer.py | 30 ++++--- scrapy/utils/httpobj.py | 11 ++- scrapy/utils/request.py | 28 +++--- scrapy/utils/response.py | 31 ++++--- scrapy/utils/trackref.py | 9 +- setup.cfg | 36 -------- 13 files changed, 154 insertions(+), 148 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 4326ca4aa..8a8065bf2 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -22,7 +22,7 @@ __all__ = [ # Scrapy and Twisted versions -__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip() +__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip() version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.')) twisted_version = (_txv.major, _txv.minor, _txv.micro) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 23ccffcd9..6e77551c6 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -3,6 +3,8 @@ Base class for Scrapy commands """ import os from optparse import OptionGroup +from typing import Any, Dict + from twisted.python import failure from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli @@ -15,7 +17,7 @@ class ScrapyCommand: crawler_process = None # default settings to be used for this command instead of global defaults - default_settings = {} + default_settings: Dict[str, Any] = {} exitcode = 0 diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 83ee074da..52118db1b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,5 +1,6 @@ import json import logging +from typing import Dict from itemadapter import is_item, ItemAdapter from w3lib.url import is_url @@ -10,6 +11,7 @@ from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output, spidercls_for_request from scrapy.exceptions import UsageError + logger = logging.getLogger(__name__) @@ -17,8 +19,8 @@ class Command(BaseRunSpiderCommand): requires_project = True spider = None - items = {} - requests = {} + items: Dict[int, list] = {} + requests: Dict[int, list] = {} first_response = None diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index db0a56e56..b47e55092 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -1,16 +1,77 @@ -import sys import re +import sys from functools import wraps from inspect import getmembers +from typing import Dict from unittest import TestCase from scrapy.http import Request -from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_spec +from scrapy.utils.spider import iterate_spider_output + + +class Contract: + """ Abstract class for contracts """ + request_cls = None + + def __init__(self, method, *args): + self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') + self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') + self.args = args + + def add_pre_hook(self, request, results): + if hasattr(self, 'pre_process'): + cb = request.callback + + @wraps(cb) + def wrapper(response, **cb_kwargs): + try: + results.startTest(self.testcase_pre) + self.pre_process(response) + results.stopTest(self.testcase_pre) + except AssertionError: + results.addFailure(self.testcase_pre, sys.exc_info()) + except Exception: + results.addError(self.testcase_pre, sys.exc_info()) + else: + results.addSuccess(self.testcase_pre) + finally: + return list(iterate_spider_output(cb(response, **cb_kwargs))) + + request.callback = wrapper + + return request + + def add_post_hook(self, request, results): + if hasattr(self, 'post_process'): + cb = request.callback + + @wraps(cb) + def wrapper(response, **cb_kwargs): + output = list(iterate_spider_output(cb(response, **cb_kwargs))) + try: + results.startTest(self.testcase_post) + self.post_process(output) + results.stopTest(self.testcase_post) + except AssertionError: + results.addFailure(self.testcase_post, sys.exc_info()) + except Exception: + results.addError(self.testcase_post, sys.exc_info()) + else: + results.addSuccess(self.testcase_post) + finally: + return output + + request.callback = wrapper + + return request + + def adjust_request_args(self, args): + return args class ContractsManager: - contracts = {} + contracts: Dict[str, Contract] = {} def __init__(self, contracts): for contract in contracts: @@ -107,66 +168,6 @@ class ContractsManager: request.errback = eb_wrapper -class Contract: - """ Abstract class for contracts """ - request_cls = None - - def __init__(self, method, *args): - self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') - self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') - self.args = args - - def add_pre_hook(self, request, results): - if hasattr(self, 'pre_process'): - cb = request.callback - - @wraps(cb) - def wrapper(response, **cb_kwargs): - try: - results.startTest(self.testcase_pre) - self.pre_process(response) - results.stopTest(self.testcase_pre) - except AssertionError: - results.addFailure(self.testcase_pre, sys.exc_info()) - except Exception: - results.addError(self.testcase_pre, sys.exc_info()) - else: - results.addSuccess(self.testcase_pre) - finally: - return list(iterate_spider_output(cb(response, **cb_kwargs))) - - request.callback = wrapper - - return request - - def add_post_hook(self, request, results): - if hasattr(self, 'post_process'): - cb = request.callback - - @wraps(cb) - def wrapper(response, **cb_kwargs): - output = list(iterate_spider_output(cb(response, **cb_kwargs))) - try: - results.startTest(self.testcase_post) - self.post_process(output) - results.stopTest(self.testcase_post) - except AssertionError: - results.addFailure(self.testcase_post, sys.exc_info()) - except Exception: - results.addError(self.testcase_post, sys.exc_info()) - else: - results.addSuccess(self.testcase_post) - finally: - return output - - request.callback = wrapper - - return request - - def adjust_request_args(self, args): - return args - - def _create_testcase(method, desc): spider = method.__self__.name diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 0c97e6999..bf4ae7b45 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,10 +1,16 @@ +import re import time -from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE +from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +# Defined in the http.cookiejar module, but undocumented: +# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 +IPV4_RE = re.compile(r"\.\d+$", re.ASCII) + + class CookieJar: def __init__(self, policy=None, check_expired_frequency=10000): self.policy = policy or DefaultCookiePolicy() diff --git a/scrapy/item.py b/scrapy/item.py index af3849302..2ccd7ad18 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -8,6 +8,7 @@ from abc import ABCMeta from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat +from typing import Dict from warnings import warn from scrapy.utils.deprecate import ScrapyDeprecationWarning @@ -75,7 +76,7 @@ class ItemMeta(_BaseItemMeta): class DictItem(MutableMapping, BaseItem): - fields = {} + fields: Dict[str, Field] = {} def __new__(cls, *args, **kwargs): if issubclass(cls, DictItem) and not issubclass(cls, Item): diff --git a/scrapy/mail.py b/scrapy/mail.py index 7d7a2c435..2a25ccd44 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -9,7 +9,7 @@ from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText -from email.utils import COMMASPACE, formatdate +from email.utils import formatdate from io import BytesIO from twisted.internet import defer, ssl @@ -21,6 +21,11 @@ from scrapy.utils.python import to_bytes logger = logging.getLogger(__name__) +# Defined in the email.utils module, but undocumented: +# https://github.com/python/cpython/blob/v3.9.0/Lib/email/utils.py#L42 +COMMASPACE = ", " + + def _to_bytes_or_none(text): if text is None: return None diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index f81041376..608c0eea5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -3,15 +3,16 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ import warnings +from typing import Tuple from urllib.parse import urlparse from w3lib.url import safe_url_string -from scrapy.http import Request, Response -from scrapy.exceptions import NotConfigured from scrapy import signals -from scrapy.utils.python import to_unicode +from scrapy.exceptions import NotConfigured +from scrapy.http import Request, Response from scrapy.utils.misc import load_object +from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url @@ -30,7 +31,8 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default" class ReferrerPolicy: - NOREFERRER_SCHEMES = LOCAL_SCHEMES + NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + name: str def referrer(self, response_url, request_url): raise NotImplementedError() @@ -88,7 +90,7 @@ class NoReferrerPolicy(ReferrerPolicy): is to be sent along with requests made from a particular request client to any origin. The header will be omitted entirely. """ - name = POLICY_NO_REFERRER + name: str = POLICY_NO_REFERRER def referrer(self, response_url, request_url): return None @@ -108,7 +110,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): This is a user agent's default behavior, if no policy is otherwise specified. """ - name = POLICY_NO_REFERRER_WHEN_DOWNGRADE + name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response_url, request_url): if not self.tls_protected(response_url) or self.tls_protected(request_url): @@ -125,7 +127,7 @@ class SameOriginPolicy(ReferrerPolicy): Cross-origin requests, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_SAME_ORIGIN + name: str = POLICY_SAME_ORIGIN def referrer(self, response_url, request_url): if self.origin(response_url) == self.origin(request_url): @@ -141,7 +143,7 @@ class OriginPolicy(ReferrerPolicy): when making both same-origin requests and cross-origin requests from a particular request client. """ - name = POLICY_ORIGIN + name: str = POLICY_ORIGIN def referrer(self, response_url, request_url): return self.origin_referrer(response_url) @@ -160,7 +162,7 @@ class StrictOriginPolicy(ReferrerPolicy): on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_STRICT_ORIGIN + name: str = POLICY_STRICT_ORIGIN def referrer(self, response_url, request_url): if ( @@ -181,7 +183,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): is sent as referrer information when making cross-origin requests from a particular request client. """ - name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN + name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response_url, request_url): origin = self.origin(response_url) @@ -208,7 +210,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN + name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response_url, request_url): origin = self.origin(response_url) @@ -234,7 +236,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): to insecure origins. Carefully consider the impact of setting such a policy for potentially sensitive documents. """ - name = POLICY_UNSAFE_URL + name: str = POLICY_UNSAFE_URL def referrer(self, response_url, request_url): return self.stripped_referrer(response_url) @@ -246,8 +248,8 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): with the addition that "Referer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') - name = POLICY_SCRAPY_DEFAULT + NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ('file', 's3') + name: str = POLICY_SCRAPY_DEFAULT _policy_classes = {p.name: p for p in ( diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index c8d4391b1..a90f1d278 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,13 +1,16 @@ """Helper functions for scrapy.http objects (Request, Response)""" -import weakref -from urllib.parse import urlparse +from typing import Union +from urllib.parse import urlparse, ParseResult +from weakref import WeakKeyDictionary + +from scrapy.http import Request, Response -_urlparse_cache = weakref.WeakKeyDictionary() +_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary() -def urlparse_cached(request_or_response): +def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult: """Return urlparse.urlparse caching the result, where the argument can be a Request or Response object """ diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 12c03d78e..66736b42f 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -4,20 +4,27 @@ scrapy.http.Request objects """ import hashlib -import weakref +from typing import Dict, Iterable, Optional, Tuple, Union from urllib.parse import urlunparse +from weakref import WeakKeyDictionary from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url +from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode -_fingerprint_cache = weakref.WeakKeyDictionary() +_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" +_fingerprint_cache = WeakKeyDictionary() -def request_fingerprint(request, include_headers=None, keep_fragments=False): +def request_fingerprint( + request: Request, + include_headers: Optional[Iterable[Union[bytes, str]]] = None, + keep_fragments: bool = False, +): """ Return the request fingerprint. @@ -49,17 +56,18 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): (for instance when handling requests with a headless browser). """ + headers: Optional[Tuple[bytes, ...]] = None if include_headers: - include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) + headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) cache = _fingerprint_cache.setdefault(request, {}) - cache_key = (include_headers, keep_fragments) + cache_key = (headers, keep_fragments) if cache_key not in cache: fp = hashlib.sha1() fp.update(to_bytes(request.method)) fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) fp.update(request.body or b'') - if include_headers: - for hdr in include_headers: + if headers: + for hdr in headers: if hdr in request.headers: fp.update(hdr) for v in request.headers.getlist(hdr): @@ -68,14 +76,14 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): return cache[cache_key] -def request_authenticate(request, username, password): +def request_authenticate(request: Request, username: str, password: str) -> None: """Autenticate the given request (in place) using the HTTP basic access authentication mechanism (RFC 2617) and the given username and password """ request.headers['Authorization'] = basic_auth_header(username, password) -def request_httprepr(request): +def request_httprepr(request: Request) -> bytes: """Return the raw HTTP representation (as bytes) of the given request. This is provided only for reference since it's not the actual stream of bytes that will be send when performing the request (that's controlled @@ -92,7 +100,7 @@ def request_httprepr(request): return s -def referer_str(request): +def referer_str(request: Request) -> Optional[str]: """ Return Referer HTTP header suitable for logging. """ referrer = request.headers.get('Referer') if referrer is None: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 99b089b6f..b3ef7b463 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,19 +3,23 @@ This module provides some useful functions for working with scrapy.http.Response objects """ import os -import weakref import webbrowser import tempfile +from typing import Any, Callable, Iterable, Optional, Tuple, Union +from weakref import WeakKeyDictionary + +import scrapy +from scrapy.http.response import Response from twisted.web import http from scrapy.utils.python import to_bytes, to_unicode from w3lib import html -_baseurl_cache = weakref.WeakKeyDictionary() +_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() -def get_base_url(response): +def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] @@ -23,10 +27,13 @@ def get_base_url(response): return _baseurl_cache[response] -_metaref_cache = weakref.WeakKeyDictionary() +_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary() -def get_meta_refresh(response, ignore_tags=('script', 'noscript')): +def get_meta_refresh( + response: "scrapy.http.response.text.TextResponse", + ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'), +) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] @@ -35,14 +42,15 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')): return _metaref_cache[response] -def response_status_message(status): +def response_status_message(status: Union[bytes, float, int, str]) -> str: """Return status code plus status text descriptive message """ - message = http.RESPONSES.get(int(status), "Unknown Status") - return f'{status} {to_unicode(message)}' + status_int = int(status) + message = http.RESPONSES.get(status_int, "Unknown Status") + return f'{status_int} {to_unicode(message)}' -def response_httprepr(response): +def response_httprepr(response: Response) -> bytes: """Return raw HTTP representation (as bytes) of the given response. This is provided only for reference, since it's not the exact stream of bytes that was received (that's not exposed by Twisted). @@ -60,7 +68,10 @@ def response_httprepr(response): return b"".join(values) -def open_in_browser(response, _openfunc=webbrowser.open): +def open_in_browser( + response: Union["scrapy.http.response.html.HtmlResponse", "scrapy.http.response.text.TextResponse"], + _openfunc: Callable[[str], Any] = webbrowser.open, +) -> Any: """Open the given response in a local web browser, populating the tag for external links to work """ diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 3e40acd69..b0c6a2424 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -9,14 +9,15 @@ and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). """ -import weakref -from time import time -from operator import itemgetter from collections import defaultdict +from operator import itemgetter +from time import time +from typing import DefaultDict +from weakref import WeakKeyDictionary NoneType = type(None) -live_refs = defaultdict(weakref.WeakKeyDictionary) +live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) class object_ref: diff --git a/setup.cfg b/setup.cfg index 0c9f6b963..89b4ec57e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,54 +10,18 @@ follow_imports = skip # FIXME: remove the following sections once the issues are solved -[mypy-scrapy] -ignore_errors = True - -[mypy-scrapy.commands] -ignore_errors = True - -[mypy-scrapy.commands.parse] -ignore_errors = True - [mypy-scrapy.downloadermiddlewares.httpproxy] ignore_errors = True -[mypy-scrapy.contracts] -ignore_errors = True - [mypy-scrapy.interfaces] ignore_errors = True -[mypy-scrapy.item] -ignore_errors = True - -[mypy-scrapy.http.cookies] -ignore_errors = True - -[mypy-scrapy.mail] -ignore_errors = True - [mypy-scrapy.pipelines.images] ignore_errors = True [mypy-scrapy.settings.default_settings] ignore_errors = True -[mypy-scrapy.spidermiddlewares.referer] -ignore_errors = True - -[mypy-scrapy.utils.httpobj] -ignore_errors = True - -[mypy-scrapy.utils.request] -ignore_errors = True - -[mypy-scrapy.utils.response] -ignore_errors = True - -[mypy-scrapy.utils.trackref] -ignore_errors = True - [mypy-tests.mocks.dummydbm] ignore_errors = True From db10aaf9eb858c8deb35e7ef96538549a8e36be0 Mon Sep 17 00:00:00 2001 From: gunadhya <6939749+gunadhya@users.noreply.github.com> Date: Thu, 3 Dec 2020 15:26:36 +0530 Subject: [PATCH 418/568] Update links in installation guide (#4899) --- docs/conf.py | 1 + docs/intro/install.rst | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 27d2b5dff..543507a46 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -283,6 +283,7 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { 'attrs': ('https://www.attrs.org/en/stable/', None), 'coverage': ('https://coverage.readthedocs.io/en/stable', None), + 'cryptography' : ('https://cryptography.io/en/latest/', None), 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), 'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None), 'pytest': ('https://docs.pytest.org/en/latest', None), diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 3bfd3bc3b..73d7ede42 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -69,10 +69,9 @@ In case of any trouble related to these dependencies, please refer to their respective installation instructions: * `lxml installation`_ -* `cryptography installation`_ +* :doc:`cryptography installation ` .. _lxml installation: https://lxml.de/installation.html -.. _cryptography installation: https://cryptography.io/en/latest/installation/ .. _intro-using-virtualenv: @@ -265,7 +264,6 @@ For details, see `Issue #2473 `_. .. _cryptography: https://cryptography.io/en/latest/ .. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/ .. _setuptools: https://pypi.python.org/pypi/setuptools -.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ .. _Scrapinghub: https://scrapinghub.com From 0dff5781bc9089884e27b0a59c1e92fc111b2817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Dec 2020 11:13:14 +0100 Subject: [PATCH 419/568] Blind attempt to fix the build of the cryptography-provided OpenSSL --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 6c39299bb..d3ff930c3 100644 --- a/tox.ini +++ b/tox.ini @@ -146,6 +146,8 @@ commands = [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython} +before_install: + - sudo apt-get -y remove libssl-dev deps = {[pinned]deps} lxml==4.0.0 From 798a818cafb21eeb62790d905d43cec678d30693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Dec 2020 13:35:40 +0100 Subject: [PATCH 420/568] Move apt-get command from Tox to Travis CI --- .travis.yml | 1 + tox.ini | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f0eafd137..055304abe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,7 @@ matrix: python: 3.8 dist: bionic install: + - apt-get -y remove libssl-dev - | if [[ ! -z "$PYPY_VERSION" ]]; then export PYPY_VERSION="pypy$PYPY_VERSION-linux64" diff --git a/tox.ini b/tox.ini index d3ff930c3..6c39299bb 100644 --- a/tox.ini +++ b/tox.ini @@ -146,8 +146,6 @@ commands = [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython} -before_install: - - sudo apt-get -y remove libssl-dev deps = {[pinned]deps} lxml==4.0.0 From 1c1255a75d315cd535dbcc5689ddda9c80c79684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Dec 2020 14:41:02 +0100 Subject: [PATCH 421/568] Use sudo for apt-get --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 055304abe..1a296c5e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ matrix: python: 3.8 dist: bionic install: - - apt-get -y remove libssl-dev + - sudo apt-get -y remove libssl-dev - | if [[ ! -z "$PYPY_VERSION" ]]; then export PYPY_VERSION="pypy$PYPY_VERSION-linux64" From 45eb099ed1937a530208eb91e28e013d56b542ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Dec 2020 15:37:02 +0100 Subject: [PATCH 422/568] =?UTF-8?q?Maybe=20it=E2=80=99s=20about=20having?= =?UTF-8?q?=20a=20newer=20libssl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1a296c5e6..880c8772b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ matrix: - env: TOXENV=asyncio-pinned python: 3.6.1 - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 + dist: bionic - env: TOXENV=py python: 3.6 @@ -42,7 +43,6 @@ matrix: python: 3.8 dist: bionic install: - - sudo apt-get -y remove libssl-dev - | if [[ ! -z "$PYPY_VERSION" ]]; then export PYPY_VERSION="pypy$PYPY_VERSION-linux64" From b83a1a6fbfe12c205c7c57830b1d4a6fc1097be0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 16 Dec 2020 18:02:47 -0300 Subject: [PATCH 423/568] Disable test under pypy --- tests/test_webclient.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index a60181a3a..f935a8689 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -4,7 +4,10 @@ Tests borrowed from the twisted.web.client tests. """ import os import shutil +import sys +from pkg_resources import parse_version +import cryptography import OpenSSL.SSL from twisted.trial import unittest from twisted.web import server, static, util, resource @@ -414,6 +417,8 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): ).addCallback(self.assertEqual, to_bytes(s)) def testPayloadDisabledCipher(self): + if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"): + self.skipTest("This does work in PyPy with cryptography<=2.3.1") s = "0123456789" * 10 settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) From 6dccf82eaa097f9e40e922d04a8b5ec9a0a9dbd7 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Tue, 22 Dec 2020 07:49:13 +1100 Subject: [PATCH 424/568] docs: fix simple typo, wihout -> without There is a small typo in scrapy/http/request/form.py. Should read `without` rather than `wihout`. --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2815303a2..7f267c800 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -160,7 +160,7 @@ def _select_value(ele, n, v): multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected - # And for select tags wihout options + # And for select tags without options o = ele.value_options return (n, o[0]) if o else (None, None) elif v is not None and multiple: From 44a7ab5bf06be3bfdf4b4a304b24f544d2f833cd Mon Sep 17 00:00:00 2001 From: Kader DJEHAF Date: Wed, 30 Dec 2020 15:22:27 +0100 Subject: [PATCH 425/568] Fix warning: Expected type 'bool', got 'int' instead (#4940) * Fix warning: Expected type 'bool', got 'int' instead * Update defer.py --- scrapy/pipelines/media.py | 2 +- scrapy/utils/defer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0a12f3e2c..0c2ee6856 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -86,7 +86,7 @@ class MediaPipeline: info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) dlist = [self._process_request(r, info, item) for r in requests] - dfd = DeferredList(dlist, consumeErrors=1) + dfd = DeferredList(dlist, consumeErrors=True) return dfd.addCallback(self.item_completed, item, info) def _process_request(self, request, info, item): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 21ba02a0b..6db9cc117 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -105,7 +105,7 @@ def process_parallel(callbacks, input, *a, **kw): callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = defer.DeferredList(dfds, fireOnOneErrback=1, consumeErrors=1) + d = defer.DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d From e494a3f73318db76d7c56e65bc632cd5875b8825 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 31 Dec 2020 11:50:15 -0300 Subject: [PATCH 426/568] protocol attribute for h2 responses --- docs/topics/request-response.rst | 2 +- scrapy/core/http2/stream.py | 2 ++ tests/test_downloader_handlers_http2.py | 7 +++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 98906992d..48f7f4a87 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -694,7 +694,7 @@ Response objects :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` :param protocol: The protocol that was used to download the response. - For instance: "HTTP/1.0", "HTTP/1.1" + For instance: "HTTP/1.0", "HTTP/1.1", "h2" :type protocol: :class:`str` .. versionadded:: 2.0.0 diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 3ae2e8db8..e345ca79a 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -15,6 +15,7 @@ from twisted.web.client import ResponseFailed from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes +from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -458,6 +459,7 @@ class Stream: request=self._request, certificate=self._protocol.metadata['certificate'], ip_address=self._protocol.metadata['ip_address'], + protocol=to_unicode(self._protocol.transport.negotiatedProtocol), ) self._deferred_response.callback(response) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 253646040..8f7f7aee0 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -22,6 +22,13 @@ class Https2TestCase(Https11TestCase): download_handler_cls = H2DownloadHandler HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "h2") + return d + @defer.inlineCallbacks def test_download_with_maxsize_very_large_file(self): with mock.patch('scrapy.core.http2.stream.logger') as logger: From 80db569aeae3e05480ed228c95332c17f67e81e4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Fri, 1 Jan 2021 19:13:39 -0300 Subject: [PATCH 427/568] Migrate CI to GitHub actions (#4924) --- .github/workflows/checks.yml | 38 ++++++++++++++ .github/workflows/main.yml | 31 ------------ .github/workflows/publish.yml | 31 ++++++++++++ .github/workflows/tests-macos.yml | 25 ++++++++++ .github/workflows/tests-ubuntu.yml | 69 ++++++++++++++++++++++++++ .github/workflows/tests-windows.yml | 32 ++++++++++++ .travis.yml | 77 ----------------------------- README.rst | 14 ++++-- tests/requirements-py3.txt | 2 +- tox.ini | 24 ++------- 10 files changed, 211 insertions(+), 132 deletions(-) create mode 100644 .github/workflows/checks.yml delete mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/tests-macos.yml create mode 100644 .github/workflows/tests-ubuntu.yml create mode 100644 .github/workflows/tests-windows.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 000000000..2748bf5fe --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,38 @@ +name: Checks +on: [push, pull_request] + +jobs: + checks: + runs-on: ubuntu-18.04 + strategy: + matrix: + include: + - python-version: 3.8 + env: + TOXENV: security + - python-version: 3.8 + env: + TOXENV: flake8 + - python-version: 3.8 + env: + TOXENV: pylint + - python-version: 3.8 + env: + TOXENV: typing + - python-version: 3.7 # Keep in sync with .readthedocs.yml + env: + TOXENV: docs + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run check + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 28771216c..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Run test suite -on: [push, pull_request] - -jobs: - test-windows: - name: "Windows Tests" - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest] - python-version: [3.7, 3.8] - env: [TOXENV: py] - include: - - os: windows-latest - python-version: 3.6 - env: - TOXENV: windows-pinned - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Run test suite - env: ${{ matrix.env }} - run: | - pip install -U tox twine wheel codecov - tox diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..aec6b8696 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,31 @@ +name: Publish +on: [push] + +jobs: + publish: + runs-on: ubuntu-18.04 + if: startsWith(github.event.ref, 'refs/tags/') + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Check Tag + id: check-release-tag + run: | + if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then + echo ::set-output name=release_tag::true + fi + + - name: Publish to PyPI + if: steps.check-release-tag.outputs.release_tag == 'true' + run: | + pip install --upgrade setuptools wheel twine + python setup.py sdist bdist_wheel + export TWINE_USERNAME=__token__ + export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} + twine upload dist/* diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml new file mode 100644 index 000000000..51d27c405 --- /dev/null +++ b/.github/workflows/tests-macos.yml @@ -0,0 +1,25 @@ +name: macOS +on: [push, pull_request] + +jobs: + tests: + runs-on: macos-10.15 + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + run: | + pip install -U tox + tox -e py + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml new file mode 100644 index 000000000..89c0334e2 --- /dev/null +++ b/.github/workflows/tests-ubuntu.yml @@ -0,0 +1,69 @@ +name: Ubuntu +on: [push, pull_request] + +jobs: + tests: + runs-on: ubuntu-18.04 + strategy: + matrix: + include: + - python-version: 3.7 + env: + TOXENV: py + - python-version: 3.8 + env: + TOXENV: py + - python-version: pypy3 + env: + TOXENV: pypy3 + PYPY_VERSION: 3.6-v7.3.1 + + # pinned deps + - python-version: 3.6.12 + env: + TOXENV: pinned + - python-version: 3.6.12 + env: + TOXENV: asyncio-pinned + - python-version: pypy3 + env: + TOXENV: pypy3-pinned + PYPY_VERSION: 3.6-v7.2.0 + + # extras + - python-version: 3.8 + env: + TOXENV: extra-deps + - python-version: 3.8 + env: + TOXENV: asyncio + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system libraries + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') + run: | + sudo apt-get update + sudo apt-get install libxml2-dev libxslt-dev + + - name: Run tests + env: ${{ matrix.env }} + run: | + if [[ ! -z "$PYPY_VERSION" ]]; then + export PYPY_VERSION="pypy$PYPY_VERSION-linux64" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi + pip install -U tox + tox + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml new file mode 100644 index 000000000..ed2e4075d --- /dev/null +++ b/.github/workflows/tests-windows.yml @@ -0,0 +1,32 @@ +name: Windows +on: [push, pull_request] + +jobs: + tests: + runs-on: windows-latest + strategy: + matrix: + include: + - python-version: 3.6 + env: + TOXENV: windows-pinned + - python-version: 3.7 + env: + TOXENV: py + - python-version: 3.8 + env: + TOXENV: py + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 880c8772b..000000000 --- a/.travis.yml +++ /dev/null @@ -1,77 +0,0 @@ -language: python -dist: xenial -branches: - only: - - master - - /^\d\.\d+$/ - - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ -matrix: - include: - - env: TOXENV=security - python: 3.8 - - env: TOXENV=flake8 - python: 3.8 - - env: TOXENV=pylint - python: 3.8 - - env: TOXENV=docs - python: 3.7 # Keep in sync with .readthedocs.yml - - env: TOXENV=typing - python: 3.8 - - - env: TOXENV=pinned - python: 3.6.1 - - env: TOXENV=asyncio-pinned - python: 3.6.1 - - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 - dist: bionic - - - env: TOXENV=py - python: 3.6 - - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 - dist: bionic - - - env: TOXENV=py - python: 3.7 - - - env: TOXENV=py PYPI_RELEASE_JOB=true - python: 3.8 - dist: bionic - - env: TOXENV=extra-deps - python: 3.8 - dist: bionic - - env: TOXENV=asyncio - python: 3.8 - dist: bionic -install: - - | - if [[ ! -z "$PYPY_VERSION" ]]; then - export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi - - pip install -U tox twine wheel codecov - -script: tox -after_success: - - codecov -notifications: - irc: - use_notice: true - skip_join: true - channels: - - irc.freenode.org#scrapy -cache: - directories: - - $HOME/.cache/pip -deploy: - provider: pypi - distributions: "sdist bdist_wheel" - user: scrapy - password: - secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA= - on: - tags: true - repo: scrapy/scrapy - condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" diff --git a/README.rst b/README.rst index a8f2ba52b..9418d270f 100644 --- a/README.rst +++ b/README.rst @@ -10,9 +10,17 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg - :target: https://travis-ci.org/scrapy/scrapy - :alt: Build Status +.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu + :alt: Ubuntu + +.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS + +.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows + :alt: Windows .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg :target: https://pypi.python.org/pypi/Scrapy diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 68b856a88..a86c4ae4f 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -14,6 +14,6 @@ uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython brotlipy # optional for HTTP compress downloader middleware tests -zstandard # optional for HTTP compress downloader middleware tests +zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tox.ini b/tox.ini index 6c39299bb..e70aef2d2 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ deps = -rtests/requirements-py3.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 - mitmproxy; python_version >= '3.7' and implementation_name != 'pypy' + mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 @@ -25,9 +25,9 @@ passenv = GCS_TEST_FILE_URI GCS_PROJECT_ID #allow tox virtualenv to upgrade pip/wheel/setuptools -download = true +download = true commands = - py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} + py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} [testenv:typing] basepython = python3 @@ -87,10 +87,6 @@ deps = botocore==1.4.87 google-cloud-storage==1.29.0 Pillow==4.0.0 -install_command = - # --use-feature=2020-resolver is required, otherwise the latest verion of - # Twisted gets installed. - pip install --use-feature=2020-resolver {opts} {packages} setenv = _SCRAPY_PINNED=true @@ -99,11 +95,9 @@ deps = {[pinned]deps} lxml==3.5.0 PyDispatcher==2.0.5 -install_command = - {[pinned]install_command} setenv = {[pinned]setenv} - + [testenv:windows-pinned] basepython = python3 deps = @@ -112,8 +106,6 @@ deps = # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 PyDispatcher==2.0.5 -install_command = - {[pinned]install_command} setenv = {[pinned]setenv} @@ -122,10 +114,6 @@ deps = {[testenv]deps} reppy robotexclusionrulesparser -install_command = - # Test --use-feature=2020-resolver for the latest version of all - # dependencies. - pip install --use-feature=2020-resolver {opts} {packages} [testenv:asyncio] commands = @@ -133,8 +121,6 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} -install_command = - {[pinned]install_command} commands = {[testenv:asyncio]commands} setenv = {[pinned]setenv} @@ -150,8 +136,6 @@ deps = {[pinned]deps} lxml==4.0.0 PyPyDispatcher==2.1.0 -install_command = - {[pinned]install_command} commands = {[testenv:pypy3]commands} setenv = {[pinned]setenv} From 0a1e2fefab04984a0f3c2b470346fd9f2ffde65b Mon Sep 17 00:00:00 2001 From: M Ikram Ullah Khan <44160462+IkramKhanNiazi@users.noreply.github.com> Date: Mon, 4 Jan 2021 18:30:23 +0500 Subject: [PATCH 428/568] Docs: fix typo in news.rst (#4942) --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 0391506c4..d9fe897ad 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -2428,7 +2428,7 @@ Bug fixes - Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`). - Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`). - Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``, - ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). + ``DOWNLOADER_MIDDLEWARES``, ``EXTENSIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). Documentation ~~~~~~~~~~~~~ From 6e7ae789f9d0b62ef5e07d16fab062ef7bdfa660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Jan 2021 11:59:38 +0100 Subject: [PATCH 429/568] Reuse the text from https://scrapy.org/ --- README.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 19faa9a87..551a06f6b 100644 --- a/README.rst +++ b/README.rst @@ -34,11 +34,10 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. -Scrapy has contributions from `many users`_ (thanks everyone!) and is sponsored -by `Scrapinghub Ltd`_. +Scrapy is maintained by `Scrapinghub`_ and `many other contributors`_. -.. _many users: https://github.com/scrapy/scrapy/graphs/contributors -.. _Scrapinghub Ltd: https://www.scrapinghub.com/ +.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapinghub: https://www.scrapinghub.com/ Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. From f30f53b3cc958770406628411db3d93c925db59e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 2 Feb 2021 15:03:20 +0100 Subject: [PATCH 430/568] =?UTF-8?q?Scrapinghub=20=E2=86=92=20Zyte?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AUTHORS | 4 +-- CODE_OF_CONDUCT.md | 2 +- README.rst | 7 ++--- docs/intro/install.rst | 1 - docs/topics/deploy.rst | 32 +++++++++++------------ docs/topics/logging.rst | 4 +-- docs/topics/practices.rst | 6 ++--- docs/topics/selectors.rst | 4 +-- scrapy/core/downloader/handlers/http11.py | 13 +++++---- 9 files changed, 38 insertions(+), 35 deletions(-) diff --git a/AUTHORS b/AUTHORS index bcaa1ecd3..9706adf42 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,8 +1,8 @@ Scrapy was brought to life by Shane Evans while hacking a scraping framework prototype for Mydeco (mydeco.com). It soon became maintained, extended and improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to -bootstrap the project. In mid-2011, Scrapinghub became the new official -maintainer. +bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new +official maintainer. Here is the list of the primary authors & contributors: diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index d1cd3e517..652460383 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -55,7 +55,7 @@ further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at opensource@scrapinghub.com. All +reported by contacting the project team at opensource@zyte.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. diff --git a/README.rst b/README.rst index bbe346522..5750e2c0f 100644 --- a/README.rst +++ b/README.rst @@ -42,10 +42,11 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. -Scrapy is maintained by `Scrapinghub`_ and `many other contributors`_. +Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other +contributors`_. .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors -.. _Scrapinghub: https://www.scrapinghub.com/ +.. _Zyte: https://www.zyte.com/ Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. @@ -95,7 +96,7 @@ Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@scrapinghub.com. +Please report unacceptable behavior to opensource@zyte.com. Companies using Scrapy ====================== diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 73d7ede42..bf919ce25 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -266,7 +266,6 @@ For details, see `Issue #2473 `_. .. _setuptools: https://pypi.python.org/pypi/setuptools .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ -.. _Scrapinghub: https://scrapinghub.com .. _Anaconda: https://docs.anaconda.com/anaconda/ .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html .. _conda-forge: https://conda-forge.org/ diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index 361914a29..961d6dc01 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -14,7 +14,7 @@ spiders come in. Popular choices for deploying Scrapy spiders are: * :ref:`Scrapyd ` (open source) -* :ref:`Scrapy Cloud ` (cloud-based) +* :ref:`Zyte Scrapy Cloud ` (cloud-based) .. _deploy-scrapyd: @@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers. .. _deploy-scrapy-cloud: -Deploying to Scrapy Cloud -========================= +Deploying to Zyte Scrapy Cloud +============================== -`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_, -the company behind Scrapy. +`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company +behind Scrapy. -Scrapy Cloud removes the need to setup and monitor servers -and provides a nice UI to manage spiders and review scraped items, -logs and stats. +Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a +nice UI to manage spiders and review scraped items, logs and stats. -To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool. -Please refer to the `Scrapy Cloud documentation`_ for more information. +To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line +tool. +Please refer to the `Zyte Scrapy Cloud documentation`_ for more information. -Scrapy Cloud is compatible with Scrapyd and one can switch between +Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between them as needed - the configuration is read from the ``scrapy.cfg`` file just like ``scrapyd-deploy``. -.. _Scrapyd: https://github.com/scrapy/scrapyd .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud +.. _Scrapyd: https://github.com/scrapy/scrapyd .. _scrapyd-client: https://github.com/scrapy/scrapyd-client -.. _shub: https://doc.scrapinghub.com/shub.html .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html -.. _Scrapinghub: https://scrapinghub.com/ +.. _shub: https://shub.readthedocs.io/en/latest/ +.. _Zyte: https://zyte.com/ +.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/ +.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 55065a1a3..c3445d40e 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -101,7 +101,7 @@ instance, which can be accessed and used like this:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['https://scrapinghub.com'] + start_urls = ['https://scrapy.org'] def parse(self, response): self.logger.info('Parse function called on %s', response.url) @@ -117,7 +117,7 @@ Python logger you want. For example:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['https://scrapinghub.com'] + start_urls = ['https://scrapy.org'] def parse(self, response): logger.info('Parse function called on %s', response.url) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cf1de1bd1..502fd5fcd 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -63,7 +63,7 @@ project as example. process = CrawlerProcess(get_project_settings()) # 'followall' is the name of one of the spiders of the project. - process.crawl('followall', domain='scrapinghub.com') + process.crawl('followall', domain='scrapy.org') process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling @@ -244,7 +244,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: super proxy that you can attach your own proxies to. * use a highly distributed downloader that circumvents bans internally, so you can just focus on parsing clean pages. One example of such downloaders is - `Crawlera`_ + `Zyte Smart Proxy Manager`_ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. @@ -254,5 +254,5 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _ProxyMesh: https://proxymesh.com/ .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders -.. _Crawlera: https://scrapinghub.com/crawlera .. _scrapoxy: https://scrapoxy.io/ +.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index b576fde91..c7ec2e0cc 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -464,10 +464,10 @@ effectively. If you are not much familiar with XPath yet, you may want to take a look first at this `XPath tutorial`_. .. note:: - Some of the tips are based on `this post from ScrapingHub's blog`_. + Some of the tips are based on `this post from Zyte's blog`_. .. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html -.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/ +.. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/ .. _topics-selectors-relative-xpaths: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a0fd837b1..513df2de9 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -303,11 +303,14 @@ class ScrapyAgent: proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams if omitConnectTunnel: - warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Crawlera, it doesn't require this mode anymore, " - "so you should update scrapy-crawlera to 1.3.0+ " - "and remove '?noconnect' from the Crawlera URL.", - ScrapyDeprecationWarning) + warnings.warn( + "Using HTTPS proxies in the noconnect mode is deprecated. " + "If you use Zyte Smart Proxy Manager (formerly Crawlera), " + "it doesn't require this mode anymore, so you should " + "update scrapy-crawlera to 1.3.0+ and remove '?noconnect' " + "from the Zyte Smart Proxy Manager URL.", + ScrapyDeprecationWarning, + ) if scheme == b'https' and not omitConnectTunnel: proxyAuth = request.headers.get(b'Proxy-Authorization', None) proxyConf = (proxyHost, proxyPort, proxyAuth) From 2ce8e0c74200481f8249bcee0f2a8a249e8fd58c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 09:09:53 +0100 Subject: [PATCH 431/568] Document the (hard-coded) maximum HTTP/2 frame size accepted from servers --- docs/topics/settings.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index b948dbfde..c7b59d582 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -689,10 +689,15 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update Scrapy currently does not support HTTP/2 Cleartext (h2c) since none of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_). + Also, Scrapy does not currently support specifying a maximum `frame size`_ + larger than the default value, 16384. Connections to servers that send a + larger frame will fail. + .. warning:: HTTP/2 support in Scrapy is experimental, and not yet recommended for production environments. Future Scrapy versions may introduce related changes without a deprecation period or warning. +.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. setting:: DOWNLOAD_TIMEOUT From d1024566d85e52f71f5591e53a3a9ee4867148ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 09:13:45 +0100 Subject: [PATCH 432/568] =?UTF-8?q?setup.py:=20Twisted=20=E2=86=92=20Twist?= =?UTF-8?q?ed[http2]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a95014c98..d1d6cc4b6 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support(): install_requires = [ - 'Twisted>=17.9.0', + 'Twisted[http2]>=17.9.0', 'cryptography>=2.0', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', From 536e749eccbdf479c9849b1868a5d95aa1210225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 09:22:02 +0100 Subject: [PATCH 433/568] HTTP/2: remove verbose protocol-handling logging --- scrapy/core/http2/protocol.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index d8d0974b8..36a51b898 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -218,7 +218,6 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.setTimeout(self.IDLE_TIMEOUT) destination = self.transport.getPeer() - logger.debug('Connection made to {}'.format(destination)) self.metadata['ip_address'] = ipaddress.ip_address(destination.host) # Initiate H2 Connection @@ -347,7 +346,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) elif isinstance(event, UnknownFrameReceived): - logger.debug('UnknownFrameReceived: frame={}'.format(event.frame)) + logger.warning(f'Unknown frame received: {event.frame}') # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated) -> None: @@ -357,15 +356,19 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def data_received(self, event: DataReceived) -> None: try: - self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) + stream = self.streams[event.stream_id] except KeyError: - logger.debug(f'Ignoring server-initiated event {event}') + pass # We ignore server-initiated events + else: + stream.receive_data(event.data, event.flow_controlled_length) def response_received(self, event: ResponseReceived) -> None: try: - self.streams[event.stream_id].receive_headers(event.headers) + stream = self.streams[event.stream_id] except KeyError: - logger.debug(f'Ignoring server-initiated event {event}') + pass # We ignore server-initiated events + else: + stream.receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: self.metadata['settings_acknowledged'] = True @@ -381,7 +384,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): try: stream = self.pop_stream(event.stream_id) except KeyError: - logger.debug(f'Ignoring server-initiated event {event}') + pass # We ignore server-initiated events else: stream.close(StreamCloseReason.ENDED, from_protocol=True) @@ -389,7 +392,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): try: stream = self.pop_stream(event.stream_id) except KeyError: - logger.debug(f'Ignoring server-initiated event {event}') + pass # We ignore server-initiated events else: stream.close(StreamCloseReason.RESET, from_protocol=True) From 1a7bde0d8e142acc06a534f6daf8910fde5de06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 10:55:11 +0100 Subject: [PATCH 434/568] Document that HTTP/2 server pushes are ignored --- docs/topics/settings.rst | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c7b59d582..05cb4a855 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -684,21 +684,28 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', } +.. warning:: + + HTTP/2 support in Scrapy is experimental, and not yet recommended for + production environments. Future Scrapy versions may introduce related + changes without a deprecation period or warning. + .. note:: - Scrapy currently does not support HTTP/2 Cleartext (h2c) since none - of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_). + Known limitations of the current HTTP/2 implementation of Scrapy include: - Also, Scrapy does not currently support specifying a maximum `frame size`_ - larger than the default value, 16384. Connections to servers that send a - larger frame will fail. + - No support for HTTP/2 Cleartext (h2c), since no major browser supports + HTTP/2 unencrypted (refer `http2 faq`_). -.. warning:: HTTP/2 support in Scrapy is experimental, and not yet recommended - for production environments. Future Scrapy versions may introduce - related changes without a deprecation period or warning. + - No setting to specify a maximum `frame size`_ larger than the default + value, 16384. Connections to servers that send a larger frame will + fail. + + - No support for `server pushes`_, which are ignored. .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption +.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 .. setting:: DOWNLOAD_TIMEOUT From 1773eaf5dc2330464cf769b7d874f24033f69707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 11:43:18 +0100 Subject: [PATCH 435/568] Move lists closer to their introducing paragraph --- docs/_static/custom.css | 10 +++ docs/conf.py | 5 +- docs/topics/exceptions.rst | 8 +- docs/topics/feed-exports.rst | 139 ++++++++++++++++++----------------- docs/topics/selectors.rst | 14 ++-- docs/topics/shell.rst | 45 ++++++------ 6 files changed, 121 insertions(+), 100 deletions(-) create mode 100644 docs/_static/custom.css diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 000000000..64f16939c --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,10 @@ +/* Move lists closer to their introducing paragraph */ +.rst-content .section ol p, .rst-content .section ul p { + margin-bottom: 0px; +} +.rst-content p + ol, .rst-content p + ul { + margin-top: -18px; /* Compensates margin-top: 24px of p */ +} +.rst-content dl p + ol, .rst-content dl p + ul { + margin-top: -6px; /* Compensates margin-top: 12px of p */ +} \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 543507a46..406c4d94a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -122,7 +122,6 @@ html_theme = 'sphinx_rtd_theme' import sphinx_rtd_theme html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # The style sheet to use for HTML and HTML Help pages. A file of that name # must exist either in Sphinx' static/ path, or in one of the custom paths # given in html_static_path. @@ -183,6 +182,10 @@ html_copy_source = True # Output file base name for HTML help builder. htmlhelp_basename = 'Scrapydoc' +html_css_files = [ + 'custom.css', +] + # Options for LaTeX output # ------------------------ diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 583a50ab8..e5264d641 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -64,10 +64,10 @@ NotConfigured This exception can be raised by some components to indicate that they will remain disabled. Those components include: - * Extensions - * Item pipelines - * Downloader middlewares - * Spider middlewares +- Extensions +- Item pipelines +- Downloader middlewares +- Spider middlewares The exception must be raised in the component's ``__init__`` method. diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 843ed25f9..e772a461c 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -21,10 +21,10 @@ Serialization formats For serializing the scraped data, the feed exports use the :ref:`Item exporters `. These formats are supported out of the box: - * :ref:`topics-feed-format-json` - * :ref:`topics-feed-format-jsonlines` - * :ref:`topics-feed-format-csv` - * :ref:`topics-feed-format-xml` +- :ref:`topics-feed-format-json` +- :ref:`topics-feed-format-jsonlines` +- :ref:`topics-feed-format-csv` +- :ref:`topics-feed-format-xml` But you can also extend the supported format through the :setting:`FEED_EXPORTERS` setting. @@ -34,54 +34,58 @@ But you can also extend the supported format through the JSON ---- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``json`` - * Exporter used: :class:`~scrapy.exporters.JsonItemExporter` - * See :ref:`this warning ` if you're using JSON with - large feeds. +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``json`` + +- Exporter used: :class:`~scrapy.exporters.JsonItemExporter` + +- See :ref:`this warning ` if you're using JSON with + large feeds. .. _topics-feed-format-jsonlines: JSON lines ---------- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines`` - * Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines`` +- Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter` .. _topics-feed-format-csv: CSV --- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv`` - * Exporter used: :class:`~scrapy.exporters.CsvItemExporter` - * To specify columns to export and their order use - :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this - option, but it is important for CSV because unlike many other export - formats CSV uses a fixed header. +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv`` + +- Exporter used: :class:`~scrapy.exporters.CsvItemExporter` + +- To specify columns to export and their order use + :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this + option, but it is important for CSV because unlike many other export + formats CSV uses a fixed header. .. _topics-feed-format-xml: XML --- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml`` - * Exporter used: :class:`~scrapy.exporters.XmlItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml`` +- Exporter used: :class:`~scrapy.exporters.XmlItemExporter` .. _topics-feed-format-pickle: Pickle ------ - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle`` - * Exporter used: :class:`~scrapy.exporters.PickleItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle`` +- Exporter used: :class:`~scrapy.exporters.PickleItemExporter` .. _topics-feed-format-marshal: Marshal ------- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal`` - * Exporter used: :class:`~scrapy.exporters.MarshalItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal`` +- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter` .. _topics-feed-storage: @@ -95,11 +99,11 @@ storage backend types which are defined by the URI scheme. The storages backends supported out of the box are: - * :ref:`topics-feed-storage-fs` - * :ref:`topics-feed-storage-ftp` - * :ref:`topics-feed-storage-s3` (requires botocore_) - * :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) - * :ref:`topics-feed-storage-stdout` +- :ref:`topics-feed-storage-fs` +- :ref:`topics-feed-storage-ftp` +- :ref:`topics-feed-storage-s3` (requires botocore_) +- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) +- :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are not available. For example, the S3 backend is only available if the botocore_ @@ -114,8 +118,8 @@ Storage URI parameters The storage URI can also contain parameters that get replaced when the feed is being created. These parameters are: - * ``%(time)s`` - gets replaced by a timestamp when the feed is being created - * ``%(name)s`` - gets replaced by the spider name +- ``%(time)s`` - gets replaced by a timestamp when the feed is being created +- ``%(name)s`` - gets replaced by the spider name Any other named parameter gets replaced by the spider attribute of the same name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id`` @@ -123,13 +127,13 @@ attribute the moment the feed is being created. Here are some examples to illustrate: - * Store in FTP using one directory per spider: +- Store in FTP using one directory per spider: - * ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json`` + - ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json`` - * Store in S3 using one directory per spider: +- Store in S3 using one directory per spider: - * ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` + - ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` .. _topics-feed-storage-backends: @@ -144,9 +148,9 @@ Local filesystem The feeds are stored in the local filesystem. - * URI scheme: ``file`` - * Example URI: ``file:///tmp/export.csv`` - * Required external libraries: none +- URI scheme: ``file`` +- Example URI: ``file:///tmp/export.csv`` +- Required external libraries: none Note that for the local filesystem storage (only) you can omit the scheme if you specify an absolute path like ``/tmp/export.csv``. This only works on Unix @@ -159,9 +163,9 @@ FTP The feeds are stored in a FTP server. - * URI scheme: ``ftp`` - * Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv`` - * Required external libraries: none +- URI scheme: ``ftp`` +- Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv`` +- Required external libraries: none FTP supports two different connection modes: `active or passive `_. Scrapy uses the passive connection @@ -178,23 +182,25 @@ S3 The feeds are stored on `Amazon S3`_. - * URI scheme: ``s3`` - * Example URIs: +- URI scheme: ``s3`` - * ``s3://mybucket/path/to/export.csv`` - * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` +- Example URIs: - * Required external libraries: `botocore`_ >= 1.4.87 + - ``s3://mybucket/path/to/export.csv`` + + - ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` + +- Required external libraries: `botocore`_ >= 1.4.87 The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: - * :setting:`AWS_ACCESS_KEY_ID` - * :setting:`AWS_SECRET_ACCESS_KEY` +- :setting:`AWS_ACCESS_KEY_ID` +- :setting:`AWS_SECRET_ACCESS_KEY` You can also define a custom ACL for exported feeds using this setting: - * :setting:`FEED_STORAGE_S3_ACL` +- :setting:`FEED_STORAGE_S3_ACL` This storage backend uses :ref:`delayed file delivery `. @@ -208,19 +214,20 @@ Google Cloud Storage (GCS) The feeds are stored on `Google Cloud Storage`_. - * URI scheme: ``gs`` - * Example URIs: +- URI scheme: ``gs`` - * ``gs://mybucket/path/to/export.csv`` +- Example URIs: - * Required external libraries: `google-cloud-storage`_. + - ``gs://mybucket/path/to/export.csv`` + +- Required external libraries: `google-cloud-storage`_. For more information about authentication, please refer to `Google Cloud documentation `_. You can set a *Project ID* and *Access Control List (ACL)* through the following settings: - * :setting:`FEED_STORAGE_GCS_ACL` - * :setting:`GCS_PROJECT_ID` +- :setting:`FEED_STORAGE_GCS_ACL` +- :setting:`GCS_PROJECT_ID` This storage backend uses :ref:`delayed file delivery `. @@ -234,9 +241,9 @@ Standard output The feeds are written to the standard output of the Scrapy process. - * URI scheme: ``stdout`` - * Example URI: ``stdout:`` - * Required external libraries: none +- URI scheme: ``stdout`` +- Example URI: ``stdout:`` +- Required external libraries: none .. _delayed-file-delivery: @@ -264,16 +271,16 @@ Settings These are the settings used for configuring the feed exports: - * :setting:`FEEDS` (mandatory) - * :setting:`FEED_EXPORT_ENCODING` - * :setting:`FEED_STORE_EMPTY` - * :setting:`FEED_EXPORT_FIELDS` - * :setting:`FEED_EXPORT_INDENT` - * :setting:`FEED_STORAGES` - * :setting:`FEED_STORAGE_FTP_ACTIVE` - * :setting:`FEED_STORAGE_S3_ACL` - * :setting:`FEED_EXPORTERS` - * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` +- :setting:`FEEDS` (mandatory) +- :setting:`FEED_EXPORT_ENCODING` +- :setting:`FEED_STORE_EMPTY` +- :setting:`FEED_EXPORT_FIELDS` +- :setting:`FEED_EXPORT_INDENT` +- :setting:`FEED_STORAGES` +- :setting:`FEED_STORAGE_FTP_ACTIVE` +- :setting:`FEED_STORAGE_S3_ACL` +- :setting:`FEED_EXPORTERS` +- :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index c7ec2e0cc..9caba5ee5 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -8,14 +8,14 @@ When you're scraping web pages, the most common task you need to perform is to extract data from the HTML source. There are several libraries available to achieve this, such as: - * `BeautifulSoup`_ is a very popular web scraping library among Python - programmers which constructs a Python object based on the structure of the - HTML code and also deals with bad markup reasonably well, but it has one - drawback: it's slow. +- `BeautifulSoup`_ is a very popular web scraping library among Python + programmers which constructs a Python object based on the structure of the + HTML code and also deals with bad markup reasonably well, but it has one + drawback: it's slow. - * `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic - API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard - library.) +- `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic + API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python + standard library.) Scrapy comes with its own mechanism for extracting data. They're called selectors because they "select" certain parts of the HTML document specified diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 0f46f1c87..b910fc453 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -95,20 +95,21 @@ convenience. Available Shortcuts ------------------- - * ``shelp()`` - print a help with the list of available objects and shortcuts +- ``shelp()`` - print a help with the list of available objects and + shortcuts - * ``fetch(url[, redirect=True])`` - fetch a new response from the given - URL and update all related objects accordingly. You can optionaly ask for - HTTP 3xx redirections to not be followed by passing ``redirect=False`` +- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL + and update all related objects accordingly. You can optionaly ask for HTTP + 3xx redirections to not be followed by passing ``redirect=False`` - * ``fetch(request)`` - fetch a new response from the given request and - update all related objects accordingly. +- ``fetch(request)`` - fetch a new response from the given request and update + all related objects accordingly. - * ``view(response)`` - open the given response in your local web browser, for - inspection. This will add a `\ tag`_ to the response body in order - for external links (such as images and style sheets) to display properly. - Note, however, that this will create a temporary file in your computer, - which won't be removed automatically. +- ``view(response)`` - open the given response in your local web browser, for + inspection. This will add a `\ tag`_ to the response body in order + for external links (such as images and style sheets) to display properly. + Note, however, that this will create a temporary file in your computer, + which won't be removed automatically. .. _ tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base @@ -122,21 +123,21 @@ content). Those objects are: - * ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object. +- ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object. - * ``spider`` - the Spider which is known to handle the URL, or a - :class:`~scrapy.spiders.Spider` object if there is no spider found for - the current URL +- ``spider`` - the Spider which is known to handle the URL, or a + :class:`~scrapy.spiders.Spider` object if there is no spider found for the + current URL - * ``request`` - a :class:`~scrapy.http.Request` object of the last fetched - page. You can modify this request using :meth:`~scrapy.http.Request.replace` - or fetch a new request (without leaving the shell) using the ``fetch`` - shortcut. +- ``request`` - a :class:`~scrapy.http.Request` object of the last fetched + page. You can modify this request using + :meth:`~scrapy.http.Request.replace` or fetch a new request (without + leaving the shell) using the ``fetch`` shortcut. - * ``response`` - a :class:`~scrapy.http.Response` object containing the last - fetched page +- ``response`` - a :class:`~scrapy.http.Response` object containing the last + fetched page - * ``settings`` - the current :ref:`Scrapy settings ` +- ``settings`` - the current :ref:`Scrapy settings ` Example of shell session ======================== From 4c801551fa10e4ff75f0768b903664f415a6a504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 21:11:46 +0100 Subject: [PATCH 436/568] Document that the bytes_received signal is not yet implemented for HTTP/2 --- docs/topics/settings.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 05cb4a855..0a4684a91 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -703,6 +703,8 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update - No support for `server pushes`_, which are ignored. + - No support for the :signal:`bytes_received` signal. + .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 From 248800328cb32f79a214289d59e927b576e21712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 21:13:43 +0100 Subject: [PATCH 437/568] Fix test_pinned_twisted_version --- tests/test_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 93e7311d2..5e63ebffb 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -36,7 +36,7 @@ class ScrapyUtilsTest(unittest.TestCase): ) config_parser = ConfigParser() config_parser.read(tox_config_file_path) - pattern = r'Twisted==([\d.]+)' + pattern = r'Twisted\[http2\]==([\d.]+)' match = re.search(pattern, config_parser['pinned']['deps']) pinned_twisted_version_string = match[1] From 0e4b291701baa8b74bd13bd47831dd040dd7173f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 21:28:04 +0100 Subject: [PATCH 438/568] HTTP/2: fix canceling a request before a connection has been established --- scrapy/core/http2/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index e345ca79a..572dbf7aa 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -459,7 +459,7 @@ class Stream: request=self._request, certificate=self._protocol.metadata['certificate'], ip_address=self._protocol.metadata['ip_address'], - protocol=to_unicode(self._protocol.transport.negotiatedProtocol), + protocol='h2', ) self._deferred_response.callback(response) From 7b11b74c77d1535f943baebbf5c794a63d147a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 4 Feb 2021 11:08:01 +0100 Subject: [PATCH 439/568] Use --use-deprecated=legacy-resolver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let’s see how test results change --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index d8e900e06..ecd3aad6e 100644 --- a/tox.ini +++ b/tox.ini @@ -27,6 +27,10 @@ passenv = GCS_PROJECT_ID #allow tox virtualenv to upgrade pip/wheel/setuptools download = true +# TODO: Remove the custom install_command below +# Temporary workaround to filter out errors caused by the insanely long time +# that it takes for the new resolver to install dependencies. +install_command=python -m pip install --use-deprecated=legacy-resolver {opts} {packages} commands = py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} From 7afcd634ab7d71a465cd9406091d371dca105789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 5 Feb 2021 13:04:54 +0100 Subject: [PATCH 440/568] Remove unused import --- scrapy/core/http2/stream.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 572dbf7aa..8a1b3e470 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -15,7 +15,6 @@ from twisted.web.client import ResponseFailed from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes -from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol From 8527b53e14e729504f6e382dd78ef6b0457fc7e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 5 Feb 2021 13:06:27 +0100 Subject: [PATCH 441/568] Revert "Use --use-deprecated=legacy-resolver" This reverts commit 7b11b74c77d1535f943baebbf5c794a63d147a13. --- tox.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tox.ini b/tox.ini index ecd3aad6e..d8e900e06 100644 --- a/tox.ini +++ b/tox.ini @@ -27,10 +27,6 @@ passenv = GCS_PROJECT_ID #allow tox virtualenv to upgrade pip/wheel/setuptools download = true -# TODO: Remove the custom install_command below -# Temporary workaround to filter out errors caused by the insanely long time -# that it takes for the new resolver to install dependencies. -install_command=python -m pip install --use-deprecated=legacy-resolver {opts} {packages} commands = py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} From 1e9b52c3e0e4eb51aaad37e796b3ee810fd919a6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 8 Feb 2021 22:02:03 +0500 Subject: [PATCH 442/568] Refactor SpiderMiddlewareManager.scrape_response. --- scrapy/core/spidermw.py | 148 +++++++++++++++++++++------------------- 1 file changed, 77 insertions(+), 71 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 763e0cdf6..289292da7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -41,86 +41,92 @@ class SpiderMiddlewareManager(MiddlewareManager): process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) - def scrape_response(self, scrape_func, response, request, spider): - - def process_spider_input(response): - for method in self.methods['process_spider_input']: - try: - result = method(response=response, spider=spider) - if result is not None: - msg = (f"Middleware {_fname(method)} must return None " - f"or raise an exception, got {type(result)}") - raise _InvalidOutput(msg) - except _InvalidOutput: - raise - except Exception: - return scrape_func(Failure(), request, spider) - return scrape_func(response, request, spider) - - def _evaluate_iterable(iterable, exception_processor_index, recover_to): + def _process_spider_input(self, scrape_func, response, request, spider): + for method in self.methods['process_spider_input']: try: - for r in iterable: - yield r + result = method(response=response, spider=spider) + if result is not None: + msg = (f"Middleware {_fname(method)} must return None " + f"or raise an exception, got {type(result)}") + raise _InvalidOutput(msg) + except _InvalidOutput: + raise + except Exception: + return scrape_func(Failure(), request, spider) + return scrape_func(response, request, spider) + + def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to): + try: + for r in iterable: + yield r + except Exception as ex: + exception_result = self._process_spider_exception(response, spider, Failure(ex), + exception_processor_index) + if isinstance(exception_result, Failure): + raise + recover_to.extend(exception_result) + + def _process_spider_exception(self, response, spider, _failure, start_index=0): + exception = _failure.value + # don't handle _InvalidOutput exception + if isinstance(exception, _InvalidOutput): + return _failure + method_list = islice(self.methods['process_spider_exception'], start_index, None) + for method_index, method in enumerate(method_list, start=start_index): + if method is None: + continue + result = method(response=response, exception=exception, spider=spider) + if _isiterable(result): + # stop exception handling by handing control over to the + # process_spider_output chain if an iterable has been returned + return self._process_spider_output(response, spider, result, method_index + 1) + elif result is None: + continue + else: + msg = (f"Middleware {_fname(method)} must return None " + f"or an iterable, got {type(result)}") + raise _InvalidOutput(msg) + return _failure + + def _process_spider_output(self, response, spider, result, start_index=0): + # items in this iterable do not need to go through the process_spider_output + # chain, they went through it already from the process_spider_exception method + recovered = MutableChain() + + method_list = islice(self.methods['process_spider_output'], start_index, None) + for method_index, method in enumerate(method_list, start=start_index): + if method is None: + continue + try: + # might fail directly if the output value is not a generator + result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result = process_spider_exception(Failure(ex), exception_processor_index) + exception_result = self._process_spider_exception(response, spider, Failure(ex), method_index + 1) if isinstance(exception_result, Failure): raise - recover_to.extend(exception_result) + return exception_result + if _isiterable(result): + result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) + else: + msg = (f"Middleware {_fname(method)} must return an " + f"iterable, got {type(result)}") + raise _InvalidOutput(msg) - def process_spider_exception(_failure, start_index=0): - exception = _failure.value - # don't handle _InvalidOutput exception - if isinstance(exception, _InvalidOutput): - return _failure - method_list = islice(self.methods['process_spider_exception'], start_index, None) - for method_index, method in enumerate(method_list, start=start_index): - if method is None: - continue - result = method(response=response, exception=exception, spider=spider) - if _isiterable(result): - # stop exception handling by handing control over to the - # process_spider_output chain if an iterable has been returned - return process_spider_output(result, method_index + 1) - elif result is None: - continue - else: - msg = (f"Middleware {_fname(method)} must return None " - f"or an iterable, got {type(result)}") - raise _InvalidOutput(msg) - return _failure + return MutableChain(result, recovered) - def process_spider_output(result, start_index=0): - # items in this iterable do not need to go through the process_spider_output - # chain, they went through it already from the process_spider_exception method - recovered = MutableChain() - - method_list = islice(self.methods['process_spider_output'], start_index, None) - for method_index, method in enumerate(method_list, start=start_index): - if method is None: - continue - try: - # might fail directly if the output value is not a generator - result = method(response=response, result=result, spider=spider) - except Exception as ex: - exception_result = process_spider_exception(Failure(ex), method_index + 1) - if isinstance(exception_result, Failure): - raise - return exception_result - if _isiterable(result): - result = _evaluate_iterable(result, method_index + 1, recovered) - else: - msg = (f"Middleware {_fname(method)} must return an " - f"iterable, got {type(result)}") - raise _InvalidOutput(msg) - - return MutableChain(result, recovered) + def _process_callback_output(self, response, spider, result): + recovered = MutableChain() + result = self._evaluate_iterable(response, spider, result, 0, recovered) + return MutableChain(self._process_spider_output(response, spider, result), recovered) + def scrape_response(self, scrape_func, response, request, spider): def process_callback_output(result): - recovered = MutableChain() - result = _evaluate_iterable(result, 0, recovered) - return MutableChain(process_spider_output(result), recovered) + return self._process_callback_output(response, spider, result) - dfd = mustbe_deferred(process_spider_input, response) + def process_spider_exception(_failure): + return self._process_spider_exception(response, spider, _failure) + + dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) return dfd From 45345ba6b508fae426223039e491dd721eaac9a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 8 Feb 2021 17:56:29 +0100 Subject: [PATCH 443/568] Use constraints.txt to limit pip resolver backtracking --- tests/constraints.txt | 9 ++++++++- tox.ini | 1 - 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/constraints.txt b/tests/constraints.txt index 5655ac2d3..3b30e6bb5 100644 --- a/tests/constraints.txt +++ b/tests/constraints.txt @@ -1 +1,8 @@ -Twisted!=18.4.0 \ No newline at end of file +# Request the latest known version or newer of some dependencies to prevent the +# pip dependency resolver from spending too much time backtracking. +attrs>=20.2.0 +Pillow>=8.0.1 +pytest>=6.2.1 +pytest-twisted>=1.13.1 +sybil>=2.0.0 +Twisted>=19.10.0 diff --git a/tox.ini b/tox.ini index d8e900e06..9908a4d51 100644 --- a/tox.ini +++ b/tox.ini @@ -67,7 +67,6 @@ commands = [pinned] deps = - -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 h2==3.2.0 From bb72bba1786bc6a725df24a01dffba2e6e2cb7c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 8 Feb 2021 21:51:57 +0100 Subject: [PATCH 444/568] tox: apply upper constraints to all non-pinned package installations --- tests/{constraints.txt => upper-constraints.txt} | 8 ++++++++ tox.ini | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) rename tests/{constraints.txt => upper-constraints.txt} (64%) diff --git a/tests/constraints.txt b/tests/upper-constraints.txt similarity index 64% rename from tests/constraints.txt rename to tests/upper-constraints.txt index 3b30e6bb5..c8c57deea 100644 --- a/tests/constraints.txt +++ b/tests/upper-constraints.txt @@ -1,8 +1,16 @@ # Request the latest known version or newer of some dependencies to prevent the # pip dependency resolver from spending too much time backtracking. attrs>=20.2.0 +Automat>=0.8.0 +itemadapter>=0.1.1 +itemloaders>=1.0.3 +lxml>=4.6.1 +parsel>=1.5.2 Pillow>=8.0.1 +pyOpenSSL>=20.0.0 pytest>=6.2.1 pytest-twisted>=1.13.1 +service_identity>=17.0.0 +six>=1.14.0 sybil>=2.0.0 Twisted>=19.10.0 diff --git a/tox.ini b/tox.ini index 9908a4d51..2dfe8987c 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,6 @@ minversion = 1.7.0 [testenv] deps = - -ctests/constraints.txt -rtests/requirements-py3.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 @@ -29,6 +28,8 @@ passenv = download = true commands = py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} +install_command = + pip install -U -ctests/upper-constraints.txt {opts} {packages} [testenv:typing] basepython = python3 @@ -90,12 +91,15 @@ deps = Pillow==4.0.0 setenv = _SCRAPY_PINNED=true +install_command = + pip install -U {opts} {packages} [testenv:pinned] deps = {[pinned]deps} lxml==3.5.0 PyDispatcher==2.0.5 +install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -107,6 +111,7 @@ deps = # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 PyDispatcher==2.0.5 +install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -123,6 +128,7 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} commands = {[testenv:asyncio]commands} +install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -138,6 +144,7 @@ deps = lxml==4.0.0 PyPyDispatcher==2.1.0 commands = {[testenv:pypy3]commands} +install_command = {[pinned]install_command} setenv = {[pinned]setenv} From 9ac5b1d021562a20b2a7d437f8bacb6754426811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 8 Feb 2021 22:31:20 +0100 Subject: [PATCH 445/568] Adjust test constraints --- tests/upper-constraints.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/upper-constraints.txt b/tests/upper-constraints.txt index c8c57deea..75f337856 100644 --- a/tests/upper-constraints.txt +++ b/tests/upper-constraints.txt @@ -2,12 +2,13 @@ # pip dependency resolver from spending too much time backtracking. attrs>=20.2.0 Automat>=0.8.0 +botocore>=1.20.3 itemadapter>=0.1.1 itemloaders>=1.0.3 lxml>=4.6.1 parsel>=1.5.2 Pillow>=8.0.1 -pyOpenSSL>=20.0.0 +pyOpenSSL>=17.5 # mitmproxy 4.0.4 pytest>=6.2.1 pytest-twisted>=1.13.1 service_identity>=17.0.0 From 15b501c0898150de686ee42dd4d78a411891e795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 10 Feb 2021 18:10:57 +0100 Subject: [PATCH 446/568] Do not force string interpolation while logging --- scrapy/core/http2/protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 36a51b898..968bfce63 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -346,7 +346,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) elif isinstance(event, UnknownFrameReceived): - logger.warning(f'Unknown frame received: {event.frame}') + logger.warning('Unknown frame received: %s', event.frame) # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated) -> None: From 54fd371481ef0ff80cc23bfab7e7f53126a31bc3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 11 Feb 2021 14:24:11 +0500 Subject: [PATCH 447/568] Skip uvloop 0.15.0+ on py36. --- tests/requirements-py3.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index a86c4ae4f..21a554624 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -9,7 +9,8 @@ pytest-twisted >= 1.11 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures -uvloop; platform_system != "Windows" +uvloop < 0.15.0; platform_system != "Windows" and python_version == '3.6' +uvloop; platform_system != "Windows" and python_version > '3.6' # optional for shell wrapper tests bpython From abbbfbbb38a87b9f7259fe47493d5810b2439963 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 12 Feb 2021 22:41:16 +0500 Subject: [PATCH 448/568] Add tests for deferred_f_from_coro_f. --- tests/test_utils_defer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index e60242a3b..7a5f458c7 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,8 +1,10 @@ +from pytest import mark from twisted.trial import unittest from twisted.internet import reactor, defer from twisted.python.failure import Failure from scrapy.utils.defer import ( + deferred_f_from_coro_f, iter_errback, mustbe_deferred, process_chain, @@ -117,3 +119,18 @@ class IterErrbackTest(unittest.TestCase): self.assertEqual(out, [0, 1, 2, 3, 4]) self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0].value, ZeroDivisionError) + + +class AsyncDefTestsuiteTest(unittest.TestCase): + @deferred_f_from_coro_f + async def test_deferred_f_from_coro_f(self): + pass + + @deferred_f_from_coro_f + async def test_deferred_f_from_coro_f_generator(self): + yield + + @mark.xfail(reason="Checks that the test is actually executed", strict=True) + @deferred_f_from_coro_f + async def test_deferred_f_from_coro_f_xfail(self): + raise Exception("This is expected to be raised") From e80f37bd3fa0f2262e898f712541bb7a12e89110 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 17 Feb 2021 16:34:29 -0300 Subject: [PATCH 449/568] Test http2 agent for unsupported scheme --- tests/test_downloader_handlers_http2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 8f7f7aee0..bee9ae75b 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -3,6 +3,7 @@ from unittest import mock from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server +from twisted.web.error import SchemeNotSupported from scrapy.core.downloader.handlers.http2 import H2DownloadHandler from scrapy.http import Request @@ -48,6 +49,12 @@ class Https2TestCase(Https11TestCase): reactor.callLater(.1, d.callback, logger) yield d + @defer.inlineCallbacks + def test_unsupported_scheme(self): + request = Request("ftp://unsupported.scheme") + d = self.download_request(request, Spider("foo")) + yield self.assertFailure(d, SchemeNotSupported) + def test_download_broken_content_cause_data_loss(self, url='broken'): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) From 4418f78941fc9d89ada9ad0436ebd0ae4ece1017 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 17 Feb 2021 18:36:52 -0300 Subject: [PATCH 450/568] Simplify check for negotiated protocol negotiatedProtocol's type is Optional[bytes] See https://github.com/twisted/twisted/blob/twisted-20.3.0/src/twisted/protocols/tls.py#L563-L587 and https://www.pyopenssl.org/en/20.0.1/api/ssl.html#OpenSSL.SSL.Connection.get_alpn_proto_negotiated Note that OpenSSL.SSL.Connection.get_next_proto_negotiated is deprecated: https://www.pyopenssl.org/en/20.0.0/changelog.html#backward-incompatible-changes --- scrapy/core/http2/protocol.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 968bfce63..9d7da14c1 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -233,13 +233,11 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def handshakeCompleted(self) -> None: """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" - negotiated_protocol = self.transport.negotiatedProtocol - if isinstance(negotiated_protocol, bytes): - negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') - if negotiated_protocol != 'h2': + protocol = self.transport.negotiatedProtocol + if protocol is not None and protocol != b"h2": # Here we have not initiated the connection yet # So, no need to send a GOAWAY frame to the remote - self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) + self._lose_connection_with_error([InvalidNegotiatedProtocol(protocol.decode("utf-8"))]) def _check_received_data(self, data: bytes) -> None: """Checks for edge cases where the connection to remote fails From 49af7c4c8b71f8643fe13c20261f97323117f980 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 18 Feb 2021 17:10:30 +0500 Subject: [PATCH 451/568] Drop pytest-twisted, use Scrapy code to install the reactor. --- conftest.py | 15 +++++++++++++++ pytest.ini | 1 - tests/requirements-py3.txt | 4 +--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/conftest.py b/conftest.py index 68b855c08..e4dd80de0 100644 --- a/conftest.py +++ b/conftest.py @@ -2,6 +2,8 @@ from pathlib import Path import pytest +from scrapy.utils.reactor import install_reactor + from tests.keys import generate_keys @@ -40,6 +42,14 @@ def pytest_collection_modifyitems(session, config, items): pass +def pytest_addoption(parser): + parser.addoption( + "--reactor", + default="default", + choices=["default", "asyncio"], + ) + + @pytest.fixture(scope='class') def reactor_pytest(request): if not request.cls: @@ -55,5 +65,10 @@ def only_asyncio(request, reactor_pytest): pytest.skip('This test is only run with --reactor=asyncio') +def pytest_configure(config): + if config.getoption("--reactor") == "asyncio": + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + + # Generate localhost certificate files, needed by some tests generate_keys() diff --git a/pytest.ini b/pytest.ini index d4deeb57c..0aae09ff5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -18,7 +18,6 @@ addopts = --ignore=docs/topics/stats.rst --ignore=docs/topics/telnetconsole.rst --ignore=docs/utils -twisted = 1 markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed flake8-max-line-length = 119 diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 21a554624..bd72c8c46 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -2,10 +2,8 @@ attrs dataclasses; python_version == '3.6' pyftpdlib -# https://github.com/pytest-dev/pytest-twisted/issues/93 -pytest != 5.4, != 5.4.1 +pytest pytest-cov -pytest-twisted >= 1.11 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures From 5fc27b1e6fc8532dc5bc08fc5abf5a0daa8ef0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 14:09:06 +0100 Subject: [PATCH 452/568] Remove RetrySpiderMixin and retry_request --- scrapy/downloadermiddlewares/retry.py | 55 --------------------------- 1 file changed, 55 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 023ab7d60..5963dacdf 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -75,61 +75,6 @@ def get_retry_request( return None -def retry_request( - request, - *, - reason, - spider, - max_retry_times=None, - priority_adjust=None, -): - new_request = get_retry_request( - request, - reason=reason, - spider=spider, - max_retry_times=max_retry_times, - priority_adjust=priority_adjust, - ) - if new_request: - return [new_request] - return [] - - -class RetrySpiderMixin: - - def get_retry_request( - self, - request, - *, - reason, - max_retry_times=None, - priority_adjust=None, - ): - return get_retry_request( - request, - reason=reason, - spider=self, - max_retry_times=max_retry_times, - priority_adjust=priority_adjust, - ) - - def retry_request( - self, - request, - *, - reason, - max_retry_times=None, - priority_adjust=None, - ): - return retry_request( - request, - reason=reason, - spider=self, - max_retry_times=max_retry_times, - priority_adjust=priority_adjust, - ) - - class RetryMiddleware: # IOError is raised by the HttpCompression middleware when trying to From 825462615a8df8e2274cf71c8a9bbea68c89262a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 14:09:48 +0100 Subject: [PATCH 453/568] =?UTF-8?q?get=5Fretry=5Frequest:=20set=20the=20de?= =?UTF-8?q?fault=20retry=20reason=20to=20=E2=80=9Cunspecified=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/downloadermiddlewares/retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5963dacdf..b8ead12ce 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -34,8 +34,8 @@ logger = logging.getLogger(__name__) def get_retry_request( request, *, - reason, spider, + reason='unspecified', max_retry_times=None, priority_adjust=None, ): From ec836dcc9290f50bad27c874cd0c25a87781735c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 14:15:28 +0100 Subject: [PATCH 454/568] Solve style issues --- scrapy/downloadermiddlewares/retry.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index b8ead12ce..5f9bc756c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -69,9 +69,12 @@ def get_retry_request( return new_request else: stats.inc_value('retry/max_reached') - logger.error("Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s", - {'request': request, 'retry_times': retry_times, 'reason': reason}, - extra={'spider': spider}) + logger.error( + "Gave up retrying %(request)s (failed %(retry_times)d times): " + "%(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider}, + ) return None From 6ab990181c6502624ceb0cca6783d99b30d30c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 14:48:03 +0100 Subject: [PATCH 455/568] Document get_retry_requests --- docs/topics/downloader-middleware.rst | 17 ++++++++++ docs/topics/settings.rst | 14 -------- scrapy/downloadermiddlewares/retry.py | 49 +++++++++++++++++++++++---- 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 6801adc9c..b539c23df 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -892,6 +892,11 @@ settings (see the settings documentation for more info): If :attr:`Request.meta ` has ``dont_retry`` key set to True, the request will be ignored by this middleware. +To retry requests from a spider callback, you can use the +:func:`get_retry_request` function: + +.. autofunction:: get_retry_request + RetryMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -932,6 +937,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because it is a common code used to indicate server overload. It is not included by default because HTTP specs say so. +.. setting:: RETRY_PRIORITY_ADJUST + +RETRY_PRIORITY_ADJUST +--------------------- + +Default: ``-1`` + +Adjust retry request priority relative to original request: + +- a positive priority adjust means higher priority. +- **a negative priority adjust (default) means lower priority.** + .. _topics-dlmw-robots: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0086a6c74..7c5e9ef6f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1188,20 +1188,6 @@ Adjust redirect request priority relative to original request: - **a positive priority adjust (default) means higher priority.** - a negative priority adjust means lower priority. -.. setting:: RETRY_PRIORITY_ADJUST - -RETRY_PRIORITY_ADJUST ---------------------- - -Default: ``-1`` - -Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware`` - -Adjust retry request priority relative to original request: - -- a positive priority adjust means higher priority. -- **a negative priority adjust (default) means lower priority.** - .. setting:: ROBOTSTXT_OBEY ROBOTSTXT_OBEY diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5f9bc756c..046e3ea71 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -39,16 +39,51 @@ def get_retry_request( max_retry_times=None, priority_adjust=None, ): + """ + Returns a new :class:`~scrapy.Request` object to retry the specified + request, or ``None`` if retries of the specified request have been + exhausted. + + For example, in a :class:`~scrapy.Spider` callback, you could use it as + follows:: + + def parse(self, response): + if not response.text: + new_request = get_retry_request( + response.request, + spider=self, + reason='empty', + ) + if new_request: + yield new_request + return + + *spider* is the :class:`~scrapy.Spider` instance which is asking for the + retry request. It is used to access the :ref:`settings ` + and :ref:`stats `, and to provide extra logging context (see + :func:`logging.debug`). + + *reason* is a string or an :class:`Exception` object that indicates the + reason why the request needs to be retried. It is used to name retry stats. + + *max_retry_times* is a number that determines the maximum number of times + that *request* can be retried. If not specified or ``None``, the number is + read from the :reqmeta:`max_retry_times` meta key of the request. If the + :reqmeta:`max_retry_times` meta key is not defined or ``None``, the number + is read from the :setting:`RETRY_TIMES` setting. + + *priority_adjust* is a number that determines how the priority of the new + request changes in relation to *request*. If not specified, the number is + read from the :setting:`RETRY_PRIORITY_ADJUST` setting. + """ settings = spider.crawler.settings stats = spider.crawler.stats retry_times = request.meta.get('retry_times', 0) + 1 - request_max_retry_times = request.meta.get( - 'max_retry_times', - max_retry_times, - ) - if request_max_retry_times is None: - request_max_retry_times = settings.getint('RETRY_TIMES') - if retry_times <= request_max_retry_times: + if max_retry_times is None: + max_retry_times = request.meta.get('max_retry_times') + if max_retry_times is None: + max_retry_times = settings.getint('RETRY_TIMES') + if retry_times <= max_retry_times: logger.debug( "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", {'request': request, 'retry_times': retry_times, 'reason': reason}, From 80f5003c88f8ee4c5529c1a6d7fc3981efef511d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 16:38:38 +0100 Subject: [PATCH 456/568] Add tests for get_retry_request --- scrapy/downloadermiddlewares/retry.py | 13 +- tests/test_downloadermiddleware_retry.py | 498 ++++++++++++++++++++--- 2 files changed, 459 insertions(+), 52 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 046e3ea71..0f24e5d28 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -10,6 +10,7 @@ Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ import logging +from inspect import isclass from twisted.internet import defer from twisted.internet.error import ( @@ -81,8 +82,8 @@ def get_retry_request( retry_times = request.meta.get('retry_times', 0) + 1 if max_retry_times is None: max_retry_times = request.meta.get('max_retry_times') - if max_retry_times is None: - max_retry_times = settings.getint('RETRY_TIMES') + if max_retry_times is None: + max_retry_times = settings.getint('RETRY_TIMES') if retry_times <= max_retry_times: logger.debug( "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", @@ -96,6 +97,8 @@ def get_retry_request( priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') new_request.priority = request.priority + priority_adjust + if isclass(reason): + reason = reason() if isinstance(reason, Exception): reason = global_object_name(reason.__class__) @@ -149,10 +152,12 @@ class RetryMiddleware: return self._retry(request, exception, spider) def _retry(self, request, reason, spider): + max_retry_times = request.meta.get('max_retry_times', self.max_retry_times) + priority_adjust = request.meta.get('priority_adjust', self.priority_adjust) return get_retry_request( request, reason=reason, spider=spider, - max_retry_times=self.max_retry_times, - priority_adjust=self.priority_adjust, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, ) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 364ce0c89..cf01a7dff 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -4,18 +4,22 @@ from twisted.internet.error import ( ConnectError, ConnectionDone, ConnectionLost, - ConnectionRefusedError, DNSLookupError, TCPTimedOutError, - TimeoutError, ) from twisted.web.client import ResponseFailed -from scrapy.downloadermiddlewares.retry import RetryMiddleware -from scrapy.spiders import Spider +from scrapy.downloadermiddlewares.retry import ( + get_retry_request, + RetryMiddleware, +) +from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +from testfixtures import LogCapture + class RetryTest(unittest.TestCase): def setUp(self): @@ -119,82 +123,480 @@ class RetryTest(unittest.TestCase): class MaxRetryTimesTest(unittest.TestCase): - def setUp(self): - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider('foo') - self.mw = RetryMiddleware.from_crawler(self.crawler) - self.mw.max_retry_times = 2 - self.invalid_url = 'http://www.scrapytest.org/invalid_url' + + invalid_url = 'http://www.scrapytest.org/invalid_url' + + def get_spider_and_middleware(self, settings=None): + crawler = get_crawler(Spider, settings or {}) + spider = crawler._create_spider('foo') + middleware = RetryMiddleware.from_crawler(crawler) + return spider, middleware def test_with_settings_zero(self): - - # SETTINGS: RETRY_TIMES = 0 - self.mw.max_retry_times = 0 - + max_retry_times = 0 + settings = {'RETRY_TIMES': max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) req = Request(self.invalid_url) - self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) + self._test_retry( + req, + DNSLookupError('foo'), + max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_metakey_zero(self): - - # SETTINGS: meta(max_retry_times) = 0 - meta_max_retry_times = 0 - - req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) - self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times) + max_retry_times = 0 + spider, middleware = self.get_spider_and_middleware() + meta = {'max_retry_times': max_retry_times} + req = Request(self.invalid_url, meta=meta) + self._test_retry( + req, + DNSLookupError('foo'), + max_retry_times, + spider=spider, + middleware=middleware, + ) def test_without_metakey(self): - - # SETTINGS: RETRY_TIMES is NON-ZERO - self.mw.max_retry_times = 5 - + max_retry_times = 5 + settings = {'RETRY_TIMES': max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) req = Request(self.invalid_url) - self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) + self._test_retry( + req, + DNSLookupError('foo'), + max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_metakey_greater(self): - - # SETINGS: RETRY_TIMES < meta(max_retry_times) - self.mw.max_retry_times = 2 meta_max_retry_times = 3 + middleware_max_retry_times = 2 req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) req2 = Request(self.invalid_url) - self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) - self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + settings = {'RETRY_TIMES': middleware_max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) + + self._test_retry( + req1, + DNSLookupError('foo'), + meta_max_retry_times, + spider=spider, + middleware=middleware, + ) + self._test_retry( + req2, + DNSLookupError('foo'), + middleware_max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_metakey_lesser(self): - - # SETINGS: RETRY_TIMES > meta(max_retry_times) - self.mw.max_retry_times = 5 meta_max_retry_times = 4 + middleware_max_retry_times = 5 req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) req2 = Request(self.invalid_url) - self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) - self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + settings = {'RETRY_TIMES': middleware_max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) + + self._test_retry( + req1, + DNSLookupError('foo'), + meta_max_retry_times, + spider=spider, + middleware=middleware, + ) + self._test_retry( + req2, + DNSLookupError('foo'), + middleware_max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_dont_retry(self): + max_retry_times = 4 + spider, middleware = self.get_spider_and_middleware() + meta = { + 'max_retry_times': max_retry_times, + 'dont_retry': True, + } + req = Request(self.invalid_url, meta=meta) + self._test_retry( + req, + DNSLookupError('foo'), + 0, + spider=spider, + middleware=middleware, + ) - # SETTINGS: meta(max_retry_times) = 4 - meta_max_retry_times = 4 - - req = Request(self.invalid_url, meta={ - 'max_retry_times': meta_max_retry_times, 'dont_retry': True - }) - - self._test_retry(req, DNSLookupError('foo'), 0) - - def _test_retry(self, req, exception, max_retry_times): + def _test_retry( + self, + req, + exception, + max_retry_times, + spider=None, + middleware=None, + ): + spider = spider or self.spider + middleware = middleware or self.mw for i in range(0, max_retry_times): - req = self.mw.process_exception(req, exception, self.spider) + req = middleware.process_exception(req, exception, spider) assert isinstance(req, Request) # discard it - req = self.mw.process_exception(req, exception, self.spider) + req = middleware.process_exception(req, exception, spider) self.assertEqual(req, None) +class GetRetryRequestTest(unittest.TestCase): + + def get_spider(self, settings=None): + crawler = get_crawler(Spider, settings or {}) + return crawler._create_spider('foo') + + def test_basic_usage(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = 1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -1) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_max_retries_reached(self): + request = Request('https://example.com') + spider = self.get_spider() + max_retry_times = 0 + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + self.assertEqual( + spider.crawler.stats.get_value('retry/max_reached'), + 1 + ) + failure_count = max_retry_times + 1 + expected_reason = "unspecified" + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) + ) + + def test_one_retry(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=1, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = 1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -1) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_two_retries(self): + spider = self.get_spider() + request = Request('https://example.com') + new_request = request + max_retry_times = 2 + for index in range(max_retry_times): + with LogCapture() as log: + new_request = get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = index+1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -expected_retry_times) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + value = spider.crawler.stats.get_value(stat) + self.assertEqual(value, expected_retry_times) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + with LogCapture() as log: + new_request = get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + self.assertEqual( + spider.crawler.stats.get_value('retry/max_reached'), + 1 + ) + failure_count = max_retry_times + 1 + expected_reason = "unspecified" + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) + ) + + def test_no_spider(self): + request = Request('https://example.com') + with self.assertRaises(TypeError): + get_retry_request(request) + + def test_max_retry_times_setting(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request, None) + + def test_max_retry_times_meta(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) + meta = {'max_retry_times': max_retry_times} + request = Request('https://example.com', meta=meta) + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request, None) + + def test_max_retry_times_argument(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) + meta = {'max_retry_times': max_retry_times + 1} + request = Request('https://example.com', meta=meta) + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + + def test_priority_adjust_setting(self): + priority_adjust = 1 + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request.priority, priority_adjust) + + def test_priority_adjust_argument(self): + priority_adjust = 1 + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust+1}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + priority_adjust=priority_adjust, + ) + self.assertEqual(new_request.priority, priority_adjust) + + def test_log_extra_retry_success(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture(attributes=('spider',)) as log: + new_request = get_retry_request( + request, + spider=spider, + ) + log.check_present(spider) + + def test_log_extra_retries_exceeded(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture(attributes=('spider',)) as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=0, + ) + log.check_present(spider) + + def test_reason_string(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = 'because' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_builtin_exception(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = NotImplementedError() + expected_reason_string = 'builtins.NotImplementedError' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_builtin_exception_class(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = NotImplementedError + expected_reason_string = 'builtins.NotImplementedError' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_custom_exception(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = IgnoreRequest() + expected_reason_string = 'scrapy.exceptions.IgnoreRequest' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_custom_exception_class(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = IgnoreRequest + expected_reason_string = 'scrapy.exceptions.IgnoreRequest' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + if __name__ == "__main__": unittest.main() From 722a33a2ac8ebc22bb7a7056598898dcb76e98a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 16:42:38 +0100 Subject: [PATCH 457/568] Fix style issues --- tests/test_downloadermiddleware_retry.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index cf01a7dff..61c0aaf2f 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -357,7 +357,7 @@ class GetRetryRequestTest(unittest.TestCase): self.assertIsInstance(new_request, Request) self.assertNotEqual(new_request, request) self.assertEqual(new_request.dont_filter, True) - expected_retry_times = index+1 + expected_retry_times = index + 1 self.assertEqual(new_request.meta['retry_times'], expected_retry_times) self.assertEqual(new_request.priority, -expected_retry_times) expected_reason = "unspecified" @@ -445,7 +445,7 @@ class GetRetryRequestTest(unittest.TestCase): def test_priority_adjust_argument(self): priority_adjust = 1 - spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust+1}) + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust + 1}) request = Request('https://example.com') new_request = get_retry_request( request, @@ -458,7 +458,7 @@ class GetRetryRequestTest(unittest.TestCase): request = Request('https://example.com') spider = self.get_spider() with LogCapture(attributes=('spider',)) as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, ) @@ -468,7 +468,7 @@ class GetRetryRequestTest(unittest.TestCase): request = Request('https://example.com') spider = self.get_spider() with LogCapture(attributes=('spider',)) as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, max_retry_times=0, @@ -480,7 +480,7 @@ class GetRetryRequestTest(unittest.TestCase): spider = self.get_spider() expected_reason = 'because' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, @@ -503,7 +503,7 @@ class GetRetryRequestTest(unittest.TestCase): expected_reason = NotImplementedError() expected_reason_string = 'builtins.NotImplementedError' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, @@ -528,7 +528,7 @@ class GetRetryRequestTest(unittest.TestCase): expected_reason = NotImplementedError expected_reason_string = 'builtins.NotImplementedError' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, @@ -553,7 +553,7 @@ class GetRetryRequestTest(unittest.TestCase): expected_reason = IgnoreRequest() expected_reason_string = 'scrapy.exceptions.IgnoreRequest' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, @@ -578,7 +578,7 @@ class GetRetryRequestTest(unittest.TestCase): expected_reason = IgnoreRequest expected_reason_string = 'scrapy.exceptions.IgnoreRequest' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, From 1f7665c4cfb955ca4e81d7bdb249cd88ada788c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 16:48:10 +0100 Subject: [PATCH 458/568] Silence a PyLint check on a mistake made for testing purposes --- tests/test_downloadermiddleware_retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 61c0aaf2f..46e525f99 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -398,7 +398,7 @@ class GetRetryRequestTest(unittest.TestCase): def test_no_spider(self): request = Request('https://example.com') with self.assertRaises(TypeError): - get_retry_request(request) + get_retry_request(request) # pylint: disable=missing-kwoa def test_max_retry_times_setting(self): max_retry_times = 0 From 6326178bc5824e8c08d84b6543de6977a16fb8d2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 22 Feb 2021 12:50:51 -0300 Subject: [PATCH 459/568] http2: acceptable protocol update, tests (#4994) --- scrapy/core/http2/protocol.py | 26 ++++++++++++++------------ tests/test_http2_client_protocol.py | 8 ++++++++ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 9d7da14c1..6ca69b23b 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -33,13 +33,16 @@ from scrapy.spiders import Spider logger = logging.getLogger(__name__) +PROTOCOL_NAME = b"h2" + + class InvalidNegotiatedProtocol(H2Error): - def __init__(self, negotiated_protocol: str) -> None: + def __init__(self, negotiated_protocol: bytes) -> None: self.negotiated_protocol = negotiated_protocol def __str__(self) -> str: - return f'InvalidNegotiatedProtocol: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}' + return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}") class RemoteTerminatedConnection(H2Error): @@ -52,7 +55,7 @@ class RemoteTerminatedConnection(H2Error): self.terminate_event = event def __str__(self) -> str: - return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address!r}' + return f'Received GOAWAY frame from {self.remote_ip_address!r}' class MethodNotAllowed405(H2Error): @@ -60,7 +63,7 @@ class MethodNotAllowed405(H2Error): self.remote_ip_address = remote_ip_address def __str__(self) -> str: - return f"MethodNotAllowed405: Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}" + return f"Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}" @implementer(IHandshakeListener) @@ -231,13 +234,12 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.transport.loseConnection() def handshakeCompleted(self) -> None: - """We close the connection with InvalidNegotiatedProtocol exception - when the connection was not made via h2 protocol""" - protocol = self.transport.negotiatedProtocol - if protocol is not None and protocol != b"h2": - # Here we have not initiated the connection yet - # So, no need to send a GOAWAY frame to the remote - self._lose_connection_with_error([InvalidNegotiatedProtocol(protocol.decode("utf-8"))]) + """ + Close the connection if it's not made via the expected protocol + """ + if self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME: + # we have not initiated the connection yet, no need to send a GOAWAY frame to the remote peer + self._lose_connection_with_error([InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)]) def _check_received_data(self, data: bytes) -> None: """Checks for edge cases where the connection to remote fails @@ -414,4 +416,4 @@ class H2ClientFactory(Factory): return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) def acceptableProtocols(self) -> List[bytes]: - return [b'h2'] + return [PROTOCOL_NAME] diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index d9ab553f0..8b2f6a11d 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,6 +5,7 @@ import re import shutil import string from ipaddress import IPv4Address +from unittest import mock from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError @@ -381,6 +382,13 @@ class Https2ClientProtocolTestCase(TestCase): 200 ) + @inlineCallbacks + def test_invalid_negotiated_protocol(self): + with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"): + request = Request(url=self.get_url('/status?n=200')) + with self.assertRaises(ResponseFailed): + yield self.make_request(request) + def test_cancel_request(self): request = Request(url=self.get_url('/get-data-html-large')) From 7605f19ec429fd3adb6a18da8e48143f99f6bfec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 23 Feb 2021 05:54:48 +0100 Subject: [PATCH 460/568] HTTP/2: test 2 concurrent requests to the same domain --- tests/test_downloader_handlers_http2.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index bee9ae75b..44d45b7d8 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -73,6 +73,21 @@ class Https2TestCase(Https11TestCase): def test_download_broken_chunked_content_allow_data_loss_via_setting(self): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + def test_concurrent_requests_same_domain(self): + spider = Spider('foo') + + request1 = Request(self.getURL('file')) + d1 = self.download_request(request1, spider) + d1.addCallback(lambda r: r.body) + d1.addCallback(self.assertEqual, b"0123456789") + + request2 = Request(self.getURL('echo'), method='POST') + d2 = self.download_request(request2, spider) + d2.addCallback(lambda r: r.headers['Content-Length']) + d2.addCallback(self.assertEqual, b"79") + + return defer.DeferredList([d1, d2]) + class Https2WrongHostnameTestCase(Https2TestCase): tls_log_message = ( From bd29f32dee445c77e7f2427a3047b7c74505efb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 23 Feb 2021 06:42:28 +0100 Subject: [PATCH 461/568] HTTP/2: do not make conn_lost_deferred optional --- scrapy/core/http2/protocol.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 6ca69b23b..1d150b7ce 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -70,7 +70,7 @@ class MethodNotAllowed405(H2Error): class H2ClientProtocol(Protocol, TimeoutMixin): IDLE_TIMEOUT = 240 - def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None: """ Arguments: uri -- URI of the base url to which HTTP/2 Connection will be made. @@ -308,8 +308,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): if not reason.check(connectionDone): self._conn_lost_errors.append(reason) - if self._conn_lost_deferred: - self._conn_lost_deferred.callback(self._conn_lost_errors) + self._conn_lost_deferred.callback(self._conn_lost_errors) for stream in self.streams.values(): if stream.metadata['request_sent']: @@ -407,7 +406,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): @implementer(IProtocolNegotiationFactory) class H2ClientFactory(Factory): - def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None: self.uri = uri self.settings = settings self.conn_lost_deferred = conn_lost_deferred From 5ba31cd1a268475db1f3cc64d4e6febb477e8f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 23 Feb 2021 11:57:33 +0100 Subject: [PATCH 462/568] HTTP/2 stream close reason handling: Use else + assert instead of elif --- scrapy/core/http2/stream.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8a1b3e470..aa44c08ce 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -431,7 +431,8 @@ class Stream: errors.insert(0, InactiveStreamClosed(self._request)) self._deferred_response.errback(ResponseFailed(errors)) - elif reason is StreamCloseReason.INVALID_HOSTNAME: + else: + assert reason is StreamCloseReason.INVALID_HOSTNAME self._deferred_response.errback(InvalidHostname( self._request, str(self._protocol.metadata['uri'].host, 'utf-8'), From 510109420733e93db0d525302555224a8364ed5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 24 Feb 2021 07:33:39 +0100 Subject: [PATCH 463/568] HTTP/2: test a CONNECT request --- tests/test_downloader_handlers_http2.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 44d45b7d8..b5a40468a 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -1,5 +1,6 @@ from unittest import mock +from pytest import mark from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server @@ -88,6 +89,14 @@ class Https2TestCase(Https11TestCase): return defer.DeferredList([d1, d2]) + @mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") + def test_connect_request(self): + request = Request(self.getURL('file'), method='CONNECT') + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b'') + return d + class Https2WrongHostnameTestCase(Https2TestCase): tls_log_message = ( From a36f952198101bfdfeadf4bec079db120809dbb1 Mon Sep 17 00:00:00 2001 From: Wehzie <39304339+Wehzie@users.noreply.github.com> Date: Wed, 24 Feb 2021 08:15:44 +0100 Subject: [PATCH 464/568] fixed typo "an quotes.json" -> "a quotes.json" (#5005) --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 9270ff42c..740e47d0c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -464,7 +464,7 @@ The simplest way to store the scraped data is by using :ref:`Feed exports scrapy crawl quotes -O quotes.json -That will generate an ``quotes.json`` file containing all scraped items, +That will generate a ``quotes.json`` file containing all scraped items, serialized in `JSON`_. The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead From 12064d799b8f15eef770a615237932b880b594a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 24 Feb 2021 10:37:38 +0100 Subject: [PATCH 465/568] HTTP/2: improve header handling --- scrapy/core/http2/stream.py | 21 ++++++++--- tests/test_downloader_handlers_http2.py | 46 +++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index aa44c08ce..8a701e7c6 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -220,11 +220,24 @@ class Stream: (':path', path), ] - for name, value in self._request.headers.items(): - headers.append((str(name, 'utf-8'), str(value[0], 'utf-8'))) + content_length = str(len(self._request.body)) + headers.append(('Content-Length', content_length)) - if b'Content-Length' not in self._request.headers.keys(): - headers.append(('Content-Length', str(len(self._request.body)))) + content_length_name = self._request.headers.normkey(b'Content-Length') + for name, values in self._request.headers.items(): + for value in values: + value = str(value, 'utf-8') + if name == content_length_name: + if value != content_length: + logger.warning( + 'Ignoring bad Content-Length header %r of request %r, ' + 'sending %r instead', + value, + self._request, + content_length, + ) + continue + headers.append((str(name, 'utf-8'), value)) return headers diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index b5a40468a..7c3db5835 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -1,6 +1,8 @@ +import json from unittest import mock from pytest import mark +from testfixtures import LogCapture from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server @@ -97,6 +99,50 @@ class Https2TestCase(Https11TestCase): d.addCallback(self.assertEqual, b'') return d + def test_custom_content_length_good(self): + request = Request(self.getURL('contentlength')) + custom_content_length = str(len(request.body)) + request.headers['Content-Length'] = custom_content_length + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.text) + d.addCallback(self.assertEqual, custom_content_length) + return d + + def test_custom_content_length_bad(self): + request = Request(self.getURL('contentlength')) + actual_content_length = str(len(request.body)) + bad_content_length = str(len(request.body)+1) + request.headers['Content-Length'] = bad_content_length + log = LogCapture() + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.text) + d.addCallback(self.assertEqual, actual_content_length) + d.addCallback( + lambda _: log.check_present( + ( + 'scrapy.core.http2.stream', + 'WARNING', + f'Ignoring bad Content-Length header ' + f'{bad_content_length!r} of request {request}, sending ' + f'{actual_content_length!r} instead', + ) + ) + ) + d.addCallback( + lambda _: log.uninstall() + ) + return d + + def test_duplicate_header(self): + request = Request(self.getURL('echo')) + header, value1, value2 = 'Custom-Header', 'foo', 'bar' + request.headers.appendlist(header, value1) + request.headers.appendlist(header, value2) + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: json.loads(r.text)['headers'][header]) + d.addCallback(self.assertEqual, [value1, value2]) + return d + class Https2WrongHostnameTestCase(Https2TestCase): tls_log_message = ( From 386e2a51ae4ed7bd53374a5cadfdf380b58284ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 24 Feb 2021 10:41:01 +0100 Subject: [PATCH 466/568] tests/test_downloader_handlers_http2.py: fix style issue --- tests/test_downloader_handlers_http2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 7c3db5835..439778014 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -111,7 +111,7 @@ class Https2TestCase(Https11TestCase): def test_custom_content_length_bad(self): request = Request(self.getURL('contentlength')) actual_content_length = str(len(request.body)) - bad_content_length = str(len(request.body)+1) + bad_content_length = str(len(request.body) + 1) request.headers['Content-Length'] = bad_content_length log = LogCapture() d = self.download_request(request, Spider('foo')) From 3894ebb1497b32959c405201f2e010292cf65098 Mon Sep 17 00:00:00 2001 From: Djiar Date: Tue, 23 Feb 2021 15:34:53 +0100 Subject: [PATCH 467/568] Refactor curl_to_request_kwargs #5001 Co-authored-by: alkazaz alkazaz@kth.se Co-authored-by: swill swill@kth.se Co-authored-by: lerjevik lerjevik@kth.se Co-authored-by: aljica aljica@kth.se --- scrapy/utils/curl.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 6660b9dc0..d8b3deaa1 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -34,6 +34,26 @@ for argument in safe_to_ignore_arguments: curl_parser.add_argument(*argument, action='store_true') +def _parse_headers_and_cookies(parsed_args): + headers = [] + cookies = {} + for header in parsed_args.headers or (): + name, val = header.split(':', 1) + name = name.strip() + val = val.strip() + if name.title() == 'Cookie': + for name, morsel in SimpleCookie(val).items(): + cookies[name] = morsel.value + else: + headers.append((name, val)) + + if parsed_args.auth: + user, password = parsed_args.auth.split(':', 1) + headers.append(('Authorization', basic_auth_header(user, password))) + + return headers, cookies + + def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): """Convert a cURL command syntax to Request kwargs. @@ -70,21 +90,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): result = {'method': method.upper(), 'url': url} - headers = [] - cookies = {} - for header in parsed_args.headers or (): - name, val = header.split(':', 1) - name = name.strip() - val = val.strip() - if name.title() == 'Cookie': - for name, morsel in SimpleCookie(val).items(): - cookies[name] = morsel.value - else: - headers.append((name, val)) - - if parsed_args.auth: - user, password = parsed_args.auth.split(':', 1) - headers.append(('Authorization', basic_auth_header(user, password))) + headers, cookies = _parse_headers_and_cookies(parsed_args) if headers: result['headers'] = headers From f689615e8d61f1f651b869b7a6284ca6ca15cde7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 24 Feb 2021 12:54:56 +0100 Subject: [PATCH 468/568] Close files in the PerYearXmlExportPipeline documentation example --- docs/topics/exporters.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 0a0a1765a..8648daded 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -50,18 +50,19 @@ value of one of their fields:: self.year_to_exporter = {} def close_spider(self, spider): - for exporter in self.year_to_exporter.values(): + for exporter, xml_file in self.year_to_exporter.values(): exporter.finish_exporting() + xml_file.close() def _exporter_for_item(self, item): adapter = ItemAdapter(item) year = adapter['year'] if year not in self.year_to_exporter: - f = open(f'{year}.xml', 'wb') - exporter = XmlItemExporter(f) + xml_file = open(f'{year}.xml', 'wb') + exporter = XmlItemExporter(xml_file) exporter.start_exporting() - self.year_to_exporter[year] = exporter - return self.year_to_exporter[year] + self.year_to_exporter[year] = (exporter, xml_file) + return self.year_to_exporter[year][0] def process_item(self, item, spider): exporter = self._exporter_for_item(item) From 7a54580679f192c4f1b66775aaddf1bee1efe448 Mon Sep 17 00:00:00 2001 From: deepang17 <47976918+deepang17@users.noreply.github.com> Date: Sun, 28 Feb 2021 15:33:09 +0530 Subject: [PATCH 469/568] DOCS:Cover scrapy-bench in the documentation --- docs/topics/benchmarking.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index b01a66188..4e53900ee 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,5 +81,4 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -In the future, more cases will be added to the benchmarking suite to cover -other common scenarios. +To use it as a project for more complex Scrapy benchmarking: https://github.com/scrapy/scrapy-bench From b25616d107d88bb195f19dca4c7e886a9ba652d3 Mon Sep 17 00:00:00 2001 From: deepang17 <47976918+deepang17@users.noreply.github.com> Date: Sun, 28 Feb 2021 16:26:46 +0530 Subject: [PATCH 470/568] DOCS: Cover scrapy-bench in the documentation --- docs/topics/benchmarking.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 4e53900ee..b15836771 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,4 +81,5 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -To use it as a project for more complex Scrapy benchmarking: https://github.com/scrapy/scrapy-bench +To use it as a project for more complex Scrapy benchmarking: +https://github.com/scrapy/scrapy-bench From 3c5668d0db4b11836bafcc91b12aba911e71f104 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Mon, 1 Mar 2021 22:00:33 -0500 Subject: [PATCH 471/568] docs: Clarify there's one extension instance per spider --- docs/topics/extensions.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 519f18b63..9e86fd0fe 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -7,8 +7,7 @@ Extensions The extensions framework provides a mechanism for inserting your own custom functionality into Scrapy. -Extensions are just regular classes that are instantiated at Scrapy startup, -when extensions are initialized. +Extensions are just regular classes. Extension settings ================== @@ -27,8 +26,8 @@ Loading & activating extensions =============================== Extensions are loaded and activated at startup by instantiating a single -instance of the extension class. Therefore, all the extension initialization -code must be performed in the class ``__init__`` method. +instance of the extension class per spider being run. All the extension +initialization code must be performed in the class ``__init__`` method. To make an extension available, add it to the :setting:`EXTENSIONS` setting in your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented From 9e62355271fa39e67b06f00f5601cdb848c7894e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 2 Mar 2021 12:09:10 -0300 Subject: [PATCH 472/568] Allow logger/stats customization in get_retry_request --- scrapy/downloadermiddlewares/retry.py | 16 ++++++--- tests/test_downloadermiddleware_retry.py | 44 ++++++++++++++++++++---- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 0f24e5d28..8955c7e4f 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -29,7 +29,8 @@ from scrapy.utils.response import response_status_message from scrapy.core.downloader.handlers.http11 import TunnelError from scrapy.utils.python import global_object_name -logger = logging.getLogger(__name__) + +retry_logger = logging.getLogger(__name__) def get_retry_request( @@ -39,6 +40,8 @@ def get_retry_request( reason='unspecified', max_retry_times=None, priority_adjust=None, + logger=retry_logger, + stats_base_key='retry', ): """ Returns a new :class:`~scrapy.Request` object to retry the specified @@ -76,6 +79,11 @@ def get_retry_request( *priority_adjust* is a number that determines how the priority of the new request changes in relation to *request*. If not specified, the number is read from the :setting:`RETRY_PRIORITY_ADJUST` setting. + + *logger* is the logging.Logger object to be used when logging messages + + *stats_base_key* is a string to be used as the base key for the + retry-related job stats """ settings = spider.crawler.settings stats = spider.crawler.stats @@ -102,11 +110,11 @@ def get_retry_request( if isinstance(reason, Exception): reason = global_object_name(reason.__class__) - stats.inc_value('retry/count') - stats.inc_value(f'retry/reason_count/{reason}') + stats.inc_value(f'{stats_base_key}/count') + stats.inc_value(f'{stats_base_key}/reason_count/{reason}') return new_request else: - stats.inc_value('retry/max_reached') + stats.inc_value(f'{stats_base_key}/max_reached') logger.error( "Gave up retrying %(request)s (failed %(retry_times)d times): " "%(reason)s", diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 46e525f99..915bd3a3e 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,4 +1,7 @@ +import logging import unittest + +from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.error import ( ConnectError, @@ -9,17 +12,12 @@ from twisted.internet.error import ( ) from twisted.web.client import ResponseFailed -from scrapy.downloadermiddlewares.retry import ( - get_retry_request, - RetryMiddleware, -) +from scrapy.downloadermiddlewares.retry import get_retry_request, RetryMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from testfixtures import LogCapture - class RetryTest(unittest.TestCase): def setUp(self): @@ -597,6 +595,40 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + def test_custom_logger(self): + logger = logging.getLogger("custom-logger") + request = Request("https://example.com") + spider = self.get_spider() + expected_reason = "because" + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + logger=logger, + ) + log.check_present( + ( + "custom-logger", + "DEBUG", + f"Retrying {request} (failed 1 times): {expected_reason}", + ) + ) + + def test_custom_stats_key(self): + request = Request("https://example.com") + spider = self.get_spider() + expected_reason = "because" + stats_key = "custom_retry" + get_retry_request( + request, + spider=spider, + reason=expected_reason, + stats_base_key=stats_key, + ) + for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + if __name__ == "__main__": unittest.main() From 36f1dbf665abd8935c3adda9a5abfef959573cdb Mon Sep 17 00:00:00 2001 From: deepang17 <47976918+deepang17@users.noreply.github.com> Date: Tue, 2 Mar 2021 22:12:44 +0530 Subject: [PATCH 473/568] DOCS: Covered scrapy-bench --- .vscode/settings.json | 3 +++ docs/topics/benchmarking.rst | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..500bc7007 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.linting.pylintEnabled": true +} \ No newline at end of file diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index b15836771..3e671365b 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,5 +81,6 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -To use it as a project for more complex Scrapy benchmarking: -https://github.com/scrapy/scrapy-bench +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench + +Use scrapy-bench_ for more complex benchmarking. \ No newline at end of file From 4fe26ae9701c95a05723a79649314da03e3ddc1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 2 Mar 2021 19:46:34 +0100 Subject: [PATCH 474/568] Limit tests to Twisted < 21 --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index e70aef2d2..69f52bd9f 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,8 @@ deps = # Extras botocore>=1.4.87 Pillow>=4.0.0 + # Twisted 21+ causes issues in tests that use skipIf + Twisted<21 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From 3d88ac605b04f4c60ca81ce509c0c3b25cfb8cd7 Mon Sep 17 00:00:00 2001 From: deepang17 <47976918+deepang17@users.noreply.github.com> Date: Tue, 9 Mar 2021 17:19:34 +0530 Subject: [PATCH 475/568] FIX: Updated benchmarking.rst --- .vscode/settings.json | 3 --- docs/topics/benchmarking.rst | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 500bc7007..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.linting.pylintEnabled": true -} \ No newline at end of file diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 3e671365b..0643df6a6 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,6 +81,6 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -.. _scrapy-bench: https://github.com/scrapy/scrapy-bench +Use scrapy-bench_ for more complex benchmarking. -Use scrapy-bench_ for more complex benchmarking. \ No newline at end of file +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench \ No newline at end of file From 3bea5e1a974b7cd99f75edb4ff728a9d7163a805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 9 Mar 2021 16:19:51 +0100 Subject: [PATCH 476/568] Remove unused _is_data_lost method --- scrapy/core/http2/stream.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8a701e7c6..c2a4b702f 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -362,14 +362,6 @@ class Stream: self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) self.close(reason) - def _is_data_lost(self) -> bool: - assert self.metadata['stream_closed_server'] - - expected_size = self._response['flow_controlled_size'] - received_body_size = int(self._response['headers'][b'Content-Length']) - - return expected_size != received_body_size - def close( self, reason: StreamCloseReason, From 0c160882306a8e33f92815cc12dc8979f04f8f5e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 11 Mar 2021 11:52:35 -0300 Subject: [PATCH 477/568] headers_received signal (#4897) --- docs/faq.rst | 11 +-- docs/topics/exceptions.rst | 7 +- docs/topics/request-response.rst | 6 +- docs/topics/signals.rst | 32 +++++++- scrapy/core/downloader/handlers/http11.py | 24 ++++++ scrapy/signals.py | 1 + tests/spiders.py | 29 ++++++++ tests/test_crawl.py | 26 ++++++- tests/test_engine.py | 85 ++++++---------------- tests/test_engine_stop_download_bytes.py | 60 +++++++++++++++ tests/test_engine_stop_download_headers.py | 56 ++++++++++++++ 11 files changed, 260 insertions(+), 77 deletions(-) create mode 100644 tests/test_engine_stop_download_bytes.py create mode 100644 tests/test_engine_stop_download_headers.py diff --git a/docs/faq.rst b/docs/faq.rst index f492dfa30..9709885f6 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -398,11 +398,12 @@ How can I cancel the download of a given response? -------------------------------------------------- In some situations, it might be useful to stop the download of a certain response. -For instance, if you only need the first part of a large response and you would like -to save resources by avoiding the download of the whole body. -In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received` -signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to -the :ref:`topics-stop-response-download` topic for additional information and examples. +For instance, sometimes you can determine whether or not you need the full contents +of a response by inspecting its headers or the first bytes of its body. In that case, +you could save resources by attaching a handler to the :class:`~scrapy.signals.bytes_received` +or :class:`~scrapy.signals.headers_received` signals and raising a +:exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the +:ref:`topics-stop-response-download` topic for additional information and examples. .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 583a50ab8..2f1517906 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -85,8 +85,8 @@ StopDownload .. exception:: StopDownload(fail=True) -Raised from a :class:`~scrapy.signals.bytes_received` signal handler to -indicate that no further bytes should be downloaded for a response. +Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received` +signal handler to indicate that no further bytes should be downloaded for a response. The ``fail`` boolean parameter controls which method will handle the resulting response: @@ -110,5 +110,6 @@ attribute. ``StopDownload(False)`` or ``StopDownload(True)`` will raise a :class:`TypeError`. -See the documentation for the :class:`~scrapy.signals.bytes_received` signal +See the documentation for the :class:`~scrapy.signals.bytes_received` and +:class:`~scrapy.signals.headers_received` signals and the :ref:`topics-stop-response-download` topic for additional information and examples. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 98906992d..37008f3e9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -432,9 +432,9 @@ The meta key is used set retry times per request. When initialized, the Stopping the download of a Response =================================== -Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a -:class:`~scrapy.signals.bytes_received` signal handler will stop the -download of a given response. See the following example:: +Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the +:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received` +signals will stop the download of a given response. See the following example:: import scrapy diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 1d99d8c28..98cfa606c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -384,6 +384,11 @@ bytes_received a possible scenario for a 25 kb response would be two signals fired with 10 kb of data, and a final one with 5 kb of data. + Handlers for this signal can stop the download of a response while it + is in progress by raising the :exc:`~scrapy.exceptions.StopDownload` + exception. Please refer to the :ref:`topics-stop-response-download` topic + for additional information and examples. + This signal does not support returning deferreds from its handlers. :param data: the data received by the download handler @@ -395,11 +400,36 @@ bytes_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.spiders.Spider` object -.. note:: Handlers of this signal can stop the download of a response while it +headers_received +~~~~~~~~~~~~~~~~ + +.. versionadded:: VERSION + +.. signal:: headers_received +.. function:: headers_received(headers, request, spider) + + Sent by the HTTP 1.1 and S3 download handlers when the response headers are + available for a given request, before downloading any additional content. + + Handlers for this signal can stop the download of a response while it is in progress by raising the :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the :ref:`topics-stop-response-download` topic for additional information and examples. + This signal does not support returning deferreds from its handlers. + + :param headers: the headers received by the download handler + :type headers: :class:`scrapy.http.headers.Headers` object + + :param body_length: expected size of the response body, in bytes + :type body_length: `int` + + :param request: the request that generated the download + :type request: :class:`~scrapy.http.Request` object + + :param spider: the spider associated with the response + :type spider: :class:`~scrapy.spiders.Spider` object + Response signals ---------------- diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 513df2de9..516a4326b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -382,6 +382,29 @@ class ScrapyAgent: return result def _cb_bodyready(self, txresponse, request): + headers_received_result = self._crawler.signals.send_catch_log( + signal=signals.headers_received, + headers=Headers(txresponse.headers.getAllRawHeaders()), + body_length=txresponse.length, + request=request, + spider=self._crawler.spider, + ) + for handler, result in headers_received_result: + if isinstance(result, Failure) and isinstance(result.value, StopDownload): + logger.debug("Download stopped for %(request)s from signal handler %(handler)s", + {"request": request, "handler": handler.__qualname__}) + txresponse._transport.stopProducing() + with suppress(AttributeError): + txresponse._transport._producer.loseConnection() + return { + "txresponse": txresponse, + "body": b"", + "flags": ["download_stopped"], + "certificate": None, + "ip_address": None, + "failure": result if result.value.fail else None, + } + # deliverBody hangs for responses without body if txresponse.length == 0: return { @@ -529,6 +552,7 @@ class _ResponseReader(protocol.Protocol): if isinstance(result, Failure) and isinstance(result.value, StopDownload): logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": self._request, "handler": handler.__qualname__}) + self.transport.stopProducing() self.transport._producer.loseConnection() failure = result if result.value.fail else None self._finish_response(flags=["download_stopped"], failure=failure) diff --git a/scrapy/signals.py b/scrapy/signals.py index c61ae6ec3..8cf2a4d93 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -17,6 +17,7 @@ request_reached_downloader = object() request_left_downloader = object() response_received = object() response_downloaded = object() +headers_received = object() bytes_received = object() item_scraped = object() item_dropped = object() diff --git a/tests/spiders.py b/tests/spiders.py index 106392ea6..7e579098a 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -390,3 +390,32 @@ class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider): def bytes_received(self, data, request, spider): self.meta["bytes_received"] = data raise StopDownload(fail=True) + + +class HeadersReceivedCallbackSpider(MetaSpider): + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + crawler.signals.connect(spider.headers_received, signals.headers_received) + return spider + + def start_requests(self): + yield Request(self.mockserver.url("/status"), errback=self.errback) + + def parse(self, response): + self.meta["response"] = response + + def errback(self, failure): + self.meta["failure"] = failure + + def headers_received(self, headers, body_length, request, spider): + self.meta["headers_received"] = headers + raise StopDownload(fail=False) + + +class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider): + + def headers_received(self, headers, body_length, request, spider): + self.meta["headers_received"] = headers + raise StopDownload(fail=True) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1083c1678..84bac9b50 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -35,6 +35,8 @@ from tests.spiders import ( DelaySpider, DuplicateStartRequestsSpider, FollowAllSpider, + HeadersReceivedCallbackSpider, + HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, ) @@ -496,7 +498,7 @@ class CrawlSpiderTestCase(TestCase): self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) @defer.inlineCallbacks - def test_stop_download_callback(self): + def test_bytes_received_stop_download_callback(self): crawler = self.runner.create_crawler(BytesReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("failure")) @@ -505,7 +507,7 @@ class CrawlSpiderTestCase(TestCase): self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length) @defer.inlineCallbacks - def test_stop_download_errback(self): + def test_bytes_received_stop_download_errback(self): crawler = self.runner.create_crawler(BytesReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("response")) @@ -518,3 +520,23 @@ class CrawlSpiderTestCase(TestCase): self.assertLess( len(crawler.spider.meta["failure"].value.response.body), crawler.spider.full_response_length) + + @defer.inlineCallbacks + def test_headers_received_stop_download_callback(self): + crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider) + yield crawler.crawl(mockserver=self.mockserver) + self.assertIsNone(crawler.spider.meta.get("failure")) + self.assertIsInstance(crawler.spider.meta["response"], Response) + self.assertEqual(crawler.spider.meta["response"].headers, crawler.spider.meta.get("headers_received")) + + @defer.inlineCallbacks + def test_headers_received_stop_download_errback(self): + crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider) + yield crawler.crawl(mockserver=self.mockserver) + self.assertIsNone(crawler.spider.meta.get("response")) + self.assertIsInstance(crawler.spider.meta["failure"], Failure) + self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) + self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) + self.assertEqual( + crawler.spider.meta["failure"].value.response.headers, + crawler.spider.meta.get("headers_received")) diff --git a/tests/test_engine.py b/tests/test_engine.py index 3629aa1aa..ef1204f94 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -19,14 +19,12 @@ from urllib.parse import urlparse import attr from itemadapter import ItemAdapter from pydispatch import dispatcher -from testfixtures import LogCapture from twisted.internet import defer, reactor from twisted.trial import unittest from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor @@ -143,6 +141,7 @@ class CrawlerRun: self.reqreached = [] self.itemerror = [] self.itemresp = [] + self.headers = {} self.bytes = defaultdict(lambda: list()) self.signals_caught = {} self.spider_class = spider_class @@ -165,6 +164,7 @@ class CrawlerRun: self.crawler = get_crawler(self.spider_class) self.crawler.signals.connect(self.item_scraped, signals.item_scraped) self.crawler.signals.connect(self.item_error, signals.item_error) + self.crawler.signals.connect(self.headers_received, signals.headers_received) self.crawler.signals.connect(self.bytes_received, signals.bytes_received) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) @@ -183,6 +183,7 @@ class CrawlerRun: if not name.startswith('_'): disconnect_all(signal) self.deferred.callback(None) + return self.crawler.stop() def geturl(self, path): return f"http://localhost:{self.portno}{path}" @@ -197,6 +198,9 @@ class CrawlerRun: def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) + def headers_received(self, headers, body_length, request, spider): + self.headers[request] = headers + def bytes_received(self, data, request, spider): self.bytes[request].append(data) @@ -220,18 +224,7 @@ class CrawlerRun: self.signals_caught[sig] = signalargs -class StopDownloadCrawlerRun(CrawlerRun): - """ - Make sure raising the StopDownload exception stops the download of the response body - """ - - def bytes_received(self, data, request, spider): - super().bytes_received(data, request, spider) - raise StopDownload(fail=False) - - class EngineTest(unittest.TestCase): - @defer.inlineCallbacks def test_crawler(self): @@ -241,8 +234,8 @@ class EngineTest(unittest.TestCase): self.run = CrawlerRun(spider) yield self.run.run() self._assert_visited_urls() - self._assert_scheduled_requests(urls_to_visit=9) - self._assert_downloaded_responses() + self._assert_scheduled_requests(count=9) + self._assert_downloaded_responses(count=9) self._assert_scraped_items() self._assert_signals_caught() self._assert_bytes_received() @@ -251,7 +244,7 @@ class EngineTest(unittest.TestCase): def test_crawler_dupefilter(self): self.run = CrawlerRun(TestDupeFilterSpider) yield self.run.run() - self._assert_scheduled_requests(urls_to_visit=8) + self._assert_scheduled_requests(count=8) self._assert_dropped_requests() @defer.inlineCallbacks @@ -267,8 +260,8 @@ class EngineTest(unittest.TestCase): urls_expected = {self.run.geturl(p) for p in must_be_visited} assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" - def _assert_scheduled_requests(self, urls_to_visit=None): - self.assertEqual(urls_to_visit, len(self.run.reqplug)) + def _assert_scheduled_requests(self, count=None): + self.assertEqual(count, len(self.run.reqplug)) paths_expected = ['/item999.html', '/item2.html', '/item1.html'] @@ -286,10 +279,10 @@ class EngineTest(unittest.TestCase): def _assert_dropped_requests(self): self.assertEqual(len(self.run.reqdropped), 1) - def _assert_downloaded_responses(self): + def _assert_downloaded_responses(self, count): # response tests - self.assertEqual(9, len(self.run.respplug)) - self.assertEqual(9, len(self.run.reqreached)) + self.assertEqual(count, len(self.run.respplug)) + self.assertEqual(count, len(self.run.reqreached)) for response, _ in self.run.respplug: if self.run.getpath(response.url) == '/item999.html': @@ -323,6 +316,13 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) + def _assert_headers_received(self): + for headers in self.run.headers.values(): + self.assertIn(b"Server", headers) + self.assertIn(b"TwistedWeb", headers[b"Server"]) + self.assertIn(b"Date", headers) + self.assertIn(b"Content-Type", headers) + def _assert_bytes_received(self): self.assertEqual(9, len(self.run.bytes)) for request, data in self.run.bytes.items(): @@ -371,6 +371,7 @@ class EngineTest(unittest.TestCase): assert signals.spider_opened in self.run.signals_caught assert signals.spider_idle in self.run.signals_caught assert signals.spider_closed in self.run.signals_caught + assert signals.headers_received in self.run.signals_caught self.assertEqual({'spider': self.run.spider}, self.run.signals_caught[signals.spider_opened]) @@ -403,48 +404,6 @@ class EngineTest(unittest.TestCase): self.assertEqual(len(e.open_spiders), 0) -class StopDownloadEngineTest(EngineTest): - - @defer.inlineCallbacks - def test_crawler(self): - for spider in TestSpider, DictItemsSpider: - self.run = StopDownloadCrawlerRun(spider) - with LogCapture() as log: - yield self.run.run() - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler" - " StopDownloadCrawlerRun.bytes_received")) - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler" - " StopDownloadCrawlerRun.bytes_received")) - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler" - " StopDownloadCrawlerRun.bytes_received")) - self._assert_visited_urls() - self._assert_scheduled_requests(urls_to_visit=9) - self._assert_downloaded_responses() - self._assert_signals_caught() - self._assert_bytes_received() - - def _assert_bytes_received(self): - self.assertEqual(9, len(self.run.bytes)) - for request, data in self.run.bytes.items(): - joined_data = b"".join(data) - self.assertTrue(len(data) == 1) # signal was fired only once - if self.run.getpath(request.url) == "/numbers": - # Received bytes are not the complete response. The exact amount depends - # on the buffer size, which can vary, so we only check that the amount - # of received bytes is strictly less than the full response. - numbers = [str(x).encode("utf8") for x in range(2**18)] - self.assertTrue(len(joined_data) < len(b"".join(numbers))) - - if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == 'runserver': start_test_site(debug=True) diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py new file mode 100644 index 000000000..0ba69e096 --- /dev/null +++ b/tests/test_engine_stop_download_bytes.py @@ -0,0 +1,60 @@ +from testfixtures import LogCapture +from twisted.internet import defer + +from scrapy.exceptions import StopDownload + +from tests.test_engine import ( + AttrsItemsSpider, + DataClassItemsSpider, + DictItemsSpider, + TestSpider, + CrawlerRun, + EngineTest, +) + + +class BytesReceivedCrawlerRun(CrawlerRun): + def bytes_received(self, data, request, spider): + super().bytes_received(data, request, spider) + raise StopDownload(fail=False) + + +class BytesReceivedEngineTest(EngineTest): + @defer.inlineCallbacks + def test_crawler(self): + for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + if spider is None: + continue + self.run = BytesReceivedCrawlerRun(spider) + with LogCapture() as log: + yield self.run.run() + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received")) + self._assert_visited_urls() + self._assert_scheduled_requests(count=9) + self._assert_downloaded_responses(count=9) + self._assert_signals_caught() + self._assert_headers_received() + self._assert_bytes_received() + + def _assert_bytes_received(self): + self.assertEqual(9, len(self.run.bytes)) + for request, data in self.run.bytes.items(): + joined_data = b"".join(data) + self.assertTrue(len(data) == 1) # signal was fired only once + if self.run.getpath(request.url) == "/numbers": + # Received bytes are not the complete response. The exact amount depends + # on the buffer size, which can vary, so we only check that the amount + # of received bytes is strictly less than the full response. + numbers = [str(x).encode("utf8") for x in range(2**18)] + self.assertTrue(len(joined_data) < len(b"".join(numbers))) diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py new file mode 100644 index 000000000..fad6643ad --- /dev/null +++ b/tests/test_engine_stop_download_headers.py @@ -0,0 +1,56 @@ +from testfixtures import LogCapture +from twisted.internet import defer + +from scrapy.exceptions import StopDownload + +from tests.test_engine import ( + AttrsItemsSpider, + DataClassItemsSpider, + DictItemsSpider, + TestSpider, + CrawlerRun, + EngineTest, +) + + +class HeadersReceivedCrawlerRun(CrawlerRun): + def headers_received(self, headers, body_length, request, spider): + super().headers_received(headers, body_length, request, spider) + raise StopDownload(fail=False) + + +class HeadersReceivedEngineTest(EngineTest): + @defer.inlineCallbacks + def test_crawler(self): + for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + if spider is None: + continue + self.run = HeadersReceivedCrawlerRun(spider) + with LogCapture() as log: + yield self.run.run() + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from" + " signal handler HeadersReceivedCrawlerRun.headers_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from signal" + " handler HeadersReceivedCrawlerRun.headers_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from" + " signal handler HeadersReceivedCrawlerRun.headers_received")) + self._assert_visited_urls() + self._assert_downloaded_responses(count=6) + self._assert_signals_caught() + self._assert_bytes_received() + self._assert_headers_received() + + def _assert_bytes_received(self): + self.assertEqual(0, len(self.run.bytes)) + + def _assert_visited_urls(self): + must_be_visited = ["/", "/redirect", "/redirected"] + urls_visited = {rp[0].url for rp in self.run.respplug} + urls_expected = {self.run.geturl(p) for p in must_be_visited} + assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" From 6e5ea7924c7e3f5dd958d13db73e04c6348c99ba Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Fri, 12 Mar 2021 11:08:41 +0600 Subject: [PATCH 478/568] Log skipped urls by length to INFO, add skipped stats --- scrapy/spidermiddlewares/urllength.py | 17 ++++++---- tests/test_spidermiddleware_urllength.py | 43 +++++++++++++++++++----- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 5be1f80cb..ee3cb9fd6 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -14,22 +14,27 @@ logger = logging.getLogger(__name__) class UrlLengthMiddleware: - def __init__(self, maxlength): + def __init__(self, maxlength, stats): self.maxlength = maxlength + self.stats = stats @classmethod - def from_settings(cls, settings): + def from_crawler(cls, crawler): + settings = crawler.settings maxlength = settings.getint('URLLENGTH_LIMIT') if not maxlength: raise NotConfigured - return cls(maxlength) + return cls(maxlength, crawler.stats) def process_spider_output(self, response, result, spider): def _filter(request): if isinstance(request, Request) and len(request.url) > self.maxlength: - logger.debug("Ignoring link (url length > %(maxlength)d): %(url)s ", - {'maxlength': self.maxlength, 'url': request.url}, - extra={'spider': spider}) + logger.info( + "Ignoring link (url length > %(maxlength)d): %(url)s ", + {'maxlength': self.maxlength, 'url': request.url}, + extra={'spider': spider} + ) + self.stats.inc_value('urllength/request_ignored_count', spider=spider) return False else: return True diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 5ef2b23fd..33c524627 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,20 +1,45 @@ from unittest import TestCase +from testfixtures import LogCapture + from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider +from scrapy.statscollectors import StatsCollector +from scrapy.utils.test import get_crawler class TestUrlLengthMiddleware(TestCase): - def test_process_spider_output(self): - res = Response('http://scrapytest.org') + def setUp(self): + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('foo') - short_url_req = Request('http://scrapytest.org/') - long_url_req = Request('http://scrapytest.org/this_is_a_long_url') - reqs = [short_url_req, long_url_req] + self.stats = StatsCollector(crawler) + self.stats.open_spider(self.spider) - mw = UrlLengthMiddleware(maxlength=25) - spider = Spider('foo') - out = list(mw.process_spider_output(res, reqs, spider)) - self.assertEqual(out, [short_url_req]) + self.maxlength = 25 + self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats) + + self.response = Response('http://scrapytest.org') + self.short_url_req = Request('http://scrapytest.org/') + self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url') + self.reqs = [self.short_url_req, self.long_url_req] + + def tearDown(self): + self.stats.close_spider(self.spider, '') + + def process_spider_output(self): + return list(self.mw.process_spider_output(self.response, self.reqs, self.spider)) + + def test_middleware_works(self): + self.assertEqual(self.process_spider_output(), [self.short_url_req]) + + def test_logging(self): + with LogCapture() as log: + self.process_spider_output() + + ric = self.stats.get_value('urllength/request_ignored_count', spider=self.spider) + self.assertEqual(ric, 1) + + self.assertIn(f'Ignoring link (url length > {self.maxlength})', str(log)) From d4b2b612551918647148013893da4cfa83fa2e7a Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Fri, 12 Mar 2021 16:59:37 +0600 Subject: [PATCH 479/568] Use from_settings for backward compatibility --- scrapy/spidermiddlewares/urllength.py | 10 ++++------ tests/test_spidermiddleware_urllength.py | 10 ++-------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index ee3cb9fd6..450d4ff40 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -14,17 +14,15 @@ logger = logging.getLogger(__name__) class UrlLengthMiddleware: - def __init__(self, maxlength, stats): + def __init__(self, maxlength): self.maxlength = maxlength - self.stats = stats @classmethod - def from_crawler(cls, crawler): - settings = crawler.settings + def from_settings(cls, settings): maxlength = settings.getint('URLLENGTH_LIMIT') if not maxlength: raise NotConfigured - return cls(maxlength, crawler.stats) + return cls(maxlength) def process_spider_output(self, response, result, spider): def _filter(request): @@ -34,7 +32,7 @@ class UrlLengthMiddleware: {'maxlength': self.maxlength, 'url': request.url}, extra={'spider': spider} ) - self.stats.inc_value('urllength/request_ignored_count', spider=spider) + spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider) return False else: return True diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 33c524627..6a72d2a8d 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -5,7 +5,6 @@ from testfixtures import LogCapture from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler @@ -14,21 +13,16 @@ class TestUrlLengthMiddleware(TestCase): def setUp(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider('foo') - - self.stats = StatsCollector(crawler) - self.stats.open_spider(self.spider) + self.stats = self.spider.crawler.stats self.maxlength = 25 - self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats) + self.mw = UrlLengthMiddleware(maxlength=self.maxlength) self.response = Response('http://scrapytest.org') self.short_url_req = Request('http://scrapytest.org/') self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url') self.reqs = [self.short_url_req, self.long_url_req] - def tearDown(self): - self.stats.close_spider(self.spider, '') - def process_spider_output(self): return list(self.mw.process_spider_output(self.response, self.reqs, self.spider)) From 0f254a6afbc3ad4a42048ea67acafdd035ba690a Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Fri, 12 Mar 2021 17:11:50 +0600 Subject: [PATCH 480/568] Test from_settings --- tests/test_spidermiddleware_urllength.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 6a72d2a8d..ee79c109f 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -6,17 +6,19 @@ from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +from scrapy.settings import Settings class TestUrlLengthMiddleware(TestCase): def setUp(self): + self.maxlength = 25 + settings = Settings({'URLLENGTH_LIMIT': self.maxlength}) + crawler = get_crawler(Spider) self.spider = crawler._create_spider('foo') self.stats = self.spider.crawler.stats - - self.maxlength = 25 - self.mw = UrlLengthMiddleware(maxlength=self.maxlength) + self.mw = UrlLengthMiddleware.from_settings(settings) self.response = Response('http://scrapytest.org') self.short_url_req = Request('http://scrapytest.org/') From c0f3ca193873cd4dbf4de730dcceb38967efcc16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 12 Mar 2021 14:02:48 +0100 Subject: [PATCH 481/568] get_retry_request: add typing information --- scrapy/downloadermiddlewares/retry.py | 28 ++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 8955c7e4f..5e49a284a 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,8 +9,8 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ -import logging -from inspect import isclass +from logging import getLogger, Logger +from typing import Optional, Union from twisted.internet import defer from twisted.internet.error import ( @@ -24,24 +24,26 @@ from twisted.internet.error import ( ) from twisted.web.client import ResponseFailed -from scrapy.exceptions import NotConfigured -from scrapy.utils.response import response_status_message from scrapy.core.downloader.handlers.http11 import TunnelError +from scrapy.exceptions import NotConfigured +from scrapy.http.request import Request +from scrapy.spiders import Spider from scrapy.utils.python import global_object_name +from scrapy.utils.response import response_status_message -retry_logger = logging.getLogger(__name__) +retry_logger = getLogger(__name__) def get_retry_request( - request, + request: Request, *, - spider, - reason='unspecified', - max_retry_times=None, - priority_adjust=None, - logger=retry_logger, - stats_base_key='retry', + spider: Spider, + reason: Union[str, Exception] = 'unspecified', + max_retry_times: Optional[int] = None, + priority_adjust: Union[int, float, None] = None, + logger: Logger = retry_logger, + stats_base_key: str = 'retry', ): """ Returns a new :class:`~scrapy.Request` object to retry the specified @@ -105,7 +107,7 @@ def get_retry_request( priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') new_request.priority = request.priority + priority_adjust - if isclass(reason): + if callable(reason): reason = reason() if isinstance(reason, Exception): reason = global_object_name(reason.__class__) From 9cc4513bd60dcebcbfc53035482eff82d2b7acc0 Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Mon, 15 Mar 2021 21:38:03 +0600 Subject: [PATCH 482/568] simpler stats access --- tests/test_spidermiddleware_urllength.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index ee79c109f..171f4ddfd 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -17,7 +17,7 @@ class TestUrlLengthMiddleware(TestCase): crawler = get_crawler(Spider) self.spider = crawler._create_spider('foo') - self.stats = self.spider.crawler.stats + self.stats = crawler.stats self.mw = UrlLengthMiddleware.from_settings(settings) self.response = Response('http://scrapytest.org') From 2f61d7cc034e6ba6ba6ae9ccfc3cf2f1021b857e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 15 Mar 2021 14:25:46 -0300 Subject: [PATCH 483/568] Remove unnecesary del statement --- scrapy/core/http2/agent.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index a142fa210..f7b0c3f99 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -70,8 +70,6 @@ class H2ConnectionPool: d = pending_requests.popleft() d.callback(conn) - del pending_requests - return conn def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None: From 42e4dbb23de238e1252eb50b059666494418588f Mon Sep 17 00:00:00 2001 From: vinayak Date: Thu, 18 Mar 2021 18:10:03 +0530 Subject: [PATCH 484/568] Support Python 3.9 (#4759) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update .travis.yml * Update .travis.yml * updage travis.yml * Make 3.9 support official * Upgrade mitmproxy for Python 3.9 * Restore the Pylint job * Undo unintended change to mitmproxy requirement * Enable Python 3.9 in GitHub Actions * Work around reppy’s Python version limitation * Disable tests in Windows / Python 3.9 due to a Twisted bug Co-authored-by: Adrián Chaves --- .github/workflows/checks.yml | 10 ++++++---- .github/workflows/publish.yml | 4 ++-- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 7 ++++++- .github/workflows/tests-windows.yml | 4 ++++ .readthedocs.yml | 6 +++++- setup.py | 1 + tox.ini | 4 +++- 8 files changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2748bf5fe..02c647da9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -7,19 +7,21 @@ jobs: strategy: matrix: include: - - python-version: 3.8 + - python-version: 3.9 env: TOXENV: security - - python-version: 3.8 + - python-version: 3.9 env: TOXENV: flake8 + # Pylint requires installing reppy, which does not support Python 3.9 + # https://github.com/seomoz/reppy/issues/122 - python-version: 3.8 env: TOXENV: pylint - - python-version: 3.8 + - python-version: 3.9 env: TOXENV: typing - - python-version: 3.7 # Keep in sync with .readthedocs.yml + - python-version: 3.8 # Keep in sync with .readthedocs.yml env: TOXENV: docs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index aec6b8696..b48066ea4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,10 +9,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Check Tag id: check-release-tag diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 51d27c405..4f8f7a19d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -6,7 +6,7 @@ jobs: runs-on: macos-10.15 strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 89c0334e2..df5ee9d69 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -13,6 +13,9 @@ jobs: - python-version: 3.8 env: TOXENV: py + - python-version: 3.9 + env: + TOXENV: py - python-version: pypy3 env: TOXENV: pypy3 @@ -31,10 +34,12 @@ jobs: PYPY_VERSION: 3.6-v7.2.0 # extras + # extra-deps includes reppy, which does not support Python 3.9 + # https://github.com/seomoz/reppy/issues/122 - python-version: 3.8 env: TOXENV: extra-deps - - python-version: 3.8 + - python-version: 3.9 env: TOXENV: asyncio diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index ed2e4075d..5459a845b 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -16,6 +16,10 @@ jobs: - python-version: 3.8 env: TOXENV: py + # https://twistedmatrix.com/trac/ticket/9990 + #- python-version: 3.9 + #env: + #TOXENV: py steps: - uses: actions/checkout@v2 diff --git a/.readthedocs.yml b/.readthedocs.yml index e4d3f02cc..80a1cd036 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,10 +3,14 @@ formats: all sphinx: configuration: docs/conf.py fail_on_warning: true + +build: + image: latest + python: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image - version: 3.7 # Keep in sync with .travis.yml + version: 3.8 # Keep in sync with .github/workflows/checks.yml install: - requirements: docs/requirements.txt - path: . diff --git a/setup.py b/setup.py index b5c42a3c2..cf9261271 100644 --- a/setup.py +++ b/setup.py @@ -85,6 +85,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', diff --git a/tox.ini b/tox.ini index 69f52bd9f..9815f80f7 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,9 @@ deps = -rtests/requirements-py3.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 - mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy' + # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe + mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' + mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 From 94201612bcad7b74c60a2a7ab70f40ba87714ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 18 Mar 2021 23:35:47 +0100 Subject: [PATCH 485/568] Simplify the get_retry_request code example --- scrapy/downloadermiddlewares/retry.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5e49a284a..2721db7cf 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -55,14 +55,12 @@ def get_retry_request( def parse(self, response): if not response.text: - new_request = get_retry_request( + new_request_or_none = get_retry_request( response.request, spider=self, reason='empty', ) - if new_request: - yield new_request - return + return new_request_or_none *spider* is the :class:`~scrapy.Spider` instance which is asking for the retry request. It is used to access the :ref:`settings ` From 8e73e1dfb51ad6a25ba4583f000d50a12718fb88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 18 Mar 2021 23:42:29 +0100 Subject: [PATCH 486/568] upper-constraints.txt: restrict botocore further --- tests/upper-constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/upper-constraints.txt b/tests/upper-constraints.txt index 75f337856..2a335e533 100644 --- a/tests/upper-constraints.txt +++ b/tests/upper-constraints.txt @@ -2,7 +2,7 @@ # pip dependency resolver from spending too much time backtracking. attrs>=20.2.0 Automat>=0.8.0 -botocore>=1.20.3 +botocore>=1.20.30 itemadapter>=0.1.1 itemloaders>=1.0.3 lxml>=4.6.1 From a390b934de4b8190499c15da115709aa6424d8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 18 Mar 2021 23:53:58 +0100 Subject: [PATCH 487/568] Do not install mitmproxy in Python 3.9 --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 86ae951b5..e0c69350e 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,8 @@ deps = # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe - mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' + # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 + #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras From 0dad0fce72266aa7b38b536f87bab26e7f233c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 19 Mar 2021 11:13:05 +0100 Subject: [PATCH 488/568] Use pip<20.3 to fix ReadTheDocs builds (#5052) --- .readthedocs.yml | 1 + docs/pip.txt | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 docs/pip.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index 80a1cd036..2d781ae81 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,5 +12,6 @@ python: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image version: 3.8 # Keep in sync with .github/workflows/checks.yml install: + - requirements: docs/pip.txt - requirements: docs/requirements.txt - path: . diff --git a/docs/pip.txt b/docs/pip.txt new file mode 100644 index 000000000..095e53a0d --- /dev/null +++ b/docs/pip.txt @@ -0,0 +1,3 @@ +# In pip 20.3-21.0, the default dependency resolver causes the build in +# ReadTheDocs to fail due to memory exhaustion or timeout. +pip<20.3 From 308a58aa275aa514397351caa263e08de2f89adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 19 Mar 2021 18:39:44 +0100 Subject: [PATCH 489/568] Update CI to support Twisted 21.2.0 (#5027) --- scrapy/pipelines/images.py | 19 +++++++++++++------ tests/test_commands.py | 3 +++ tests/test_crawler.py | 4 ++++ tests/test_pipeline_crawl.py | 11 +++++++++++ tests/test_pipeline_images.py | 14 ++++++++++---- tests/test_pipeline_media.py | 11 +++++++++++ tox.ini | 4 +--- 7 files changed, 53 insertions(+), 13 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index aafd1d8b2..e3ab23ea5 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -9,9 +9,8 @@ from contextlib import suppress from io import BytesIO from itemadapter import ItemAdapter -from PIL import Image -from scrapy.exceptions import DropItem +from scrapy.exceptions import DropItem, NotConfigured from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline @@ -45,6 +44,14 @@ class ImagesPipeline(FilesPipeline): DEFAULT_IMAGES_RESULT_FIELD = 'images' def __init__(self, store_uri, download_func=None, settings=None): + try: + from PIL import Image + self._Image = Image + except ImportError: + raise NotConfigured( + 'ImagesPipeline requires installing Pillow 4.0.0 or later' + ) + super().__init__(store_uri, settings=settings, download_func=download_func) if isinstance(settings, dict) or settings is None: @@ -121,7 +128,7 @@ class ImagesPipeline(FilesPipeline): def get_images(self, response, request, info, *, item=None): path = self.file_path(request, response=response, info=info, item=item) - orig_image = Image.open(BytesIO(response.body)) + orig_image = self._Image.open(BytesIO(response.body)) width, height = orig_image.size if width < self.min_width or height < self.min_height: @@ -139,12 +146,12 @@ class ImagesPipeline(FilesPipeline): def convert_image(self, image, size=None): if image.format == 'PNG' and image.mode == 'RGBA': - background = Image.new('RGBA', image.size, (255, 255, 255)) + background = self._Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert('RGB') elif image.mode == 'P': image = image.convert("RGBA") - background = Image.new('RGBA', image.size, (255, 255, 255)) + background = self._Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert('RGB') elif image.mode != 'RGB': @@ -152,7 +159,7 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() - image.thumbnail(size, Image.ANTIALIAS) + image.thumbnail(size, self._Image.ANTIALIAS) buf = BytesIO() image.save(buf, 'JPEG') diff --git a/tests/test_commands.py b/tests/test_commands.py index d3ac05eac..eec1f02ee 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -17,6 +17,8 @@ from threading import Timer from unittest import skipIf from pytest import mark +from twisted import version as twisted_version +from twisted.python.versions import Version from twisted.trial import unittest import scrapy @@ -630,6 +632,7 @@ class MySpider(scrapy.Spider): @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') def test_custom_asyncio_loop_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ab113710d..dec517bb6 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -8,7 +8,9 @@ from unittest import skipIf from pytest import raises, mark from testfixtures import LogCapture +from twisted import version as twisted_version from twisted.internet import defer +from twisted.python.versions import Version from twisted.trial import unittest import scrapy @@ -358,6 +360,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") self.assertIn("Spider closed (finished)", log) @@ -366,6 +369,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly") @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows") + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 55fcfa7ba..f49fda701 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -180,7 +180,18 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) +try: + from PIL import Image # noqa: imported just to check for the import error +except ImportError: + skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' +else: + skip_pillow = None + + class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): + + skip = skip_pillow + pipeline_class = 'scrapy.pipelines.images.ImagesPipeline' store_setting_key = 'IMAGES_STORE' media_key = 'images' diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index ad138a2dc..c69cd0e4a 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -23,15 +23,16 @@ except ImportError: dataclass_field = None -skip = False try: from PIL import Image except ImportError: - skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' + skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' else: encoders = {'jpeg_encoder', 'jpeg_decoder'} if not encoders.issubset(set(Image.core.__dict__)): - skip = 'Missing JPEG encoders' + skip_pillow = 'Missing JPEG encoders' + else: + skip_pillow = None def _mocked_download_func(request, info): @@ -41,7 +42,7 @@ def _mocked_download_func(request, info): class ImagesPipelineTestCase(unittest.TestCase): - skip = skip + skip = skip_pillow def setUp(self): self.tempdir = mkdtemp() @@ -137,6 +138,8 @@ class DeprecatedImagesPipeline(ImagesPipeline): class ImagesPipelineTestCaseFieldsMixin: + skip = skip_pillow + def test_item_fields_default(self): url = 'http://www.example.com/images/1.jpg' item = self.item_class(name='item1', image_urls=[url]) @@ -221,6 +224,9 @@ class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, u class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): + + skip = skip_pillow + img_cls_attribute_names = [ # Pipeline attribute names with corresponding setting names. ("EXPIRES", "IMAGES_EXPIRES"), diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 6afd47497..893d43052 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,3 +1,5 @@ +from typing import Optional + from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure @@ -17,6 +19,14 @@ from scrapy.utils.signal import disconnect_all from scrapy import signals +try: + from PIL import Image # noqa: imported just to check for the import error +except ImportError: + skip_pillow: Optional[str] = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' +else: + skip_pillow = None + + def _mocked_download_func(request, info): response = request.meta.get('response') return response() if callable(response) else response @@ -379,6 +389,7 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): + skip = skip_pillow def setUp(self): self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func) diff --git a/tox.ini b/tox.ini index e0c69350e..6907c8906 100644 --- a/tox.ini +++ b/tox.ini @@ -19,9 +19,6 @@ deps = mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 - Pillow>=4.0.0 - # Twisted 21+ causes issues in tests that use skipIf - Twisted[http2]>=17.9.0,<21 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -124,6 +121,7 @@ deps = {[testenv]deps} reppy robotexclusionrulesparser + Pillow>=4.0.0 [testenv:asyncio] commands = From 2973d8d51abbda4839981c192a366e907e2ad39e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Mar 2021 11:24:10 +0100 Subject: [PATCH 490/568] Remove unnecessary reference to private parsel.Selector._default_type (#5006) --- scrapy/selector/unified.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index a25871433..08f08e8d7 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -69,7 +69,7 @@ class Selector(_ParselSelector, object_ref): raise ValueError(f'{self.__class__.__name__}.__init__() received ' 'both response and text') - st = _st(response, type or self._default_type) + st = _st(response, type) if text is not None: response = _response_from_text(text, st) From ec5a7918ec2d8888ba356f9e3295dcd1ee935884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Mar 2021 11:25:40 +0100 Subject: [PATCH 491/568] Include Content-Length in HTTP/1.1 responses (#5057) --- scrapy/core/downloader/handlers/http11.py | 13 +++++++++++-- tests/test_downloader_handlers.py | 7 +++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9cdadb27f..1f82751fd 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -358,10 +358,19 @@ class ScrapyAgent: request.meta['download_latency'] = time() - start_time return result + @staticmethod + def _headers_from_twisted_response(response): + headers = Headers() + if response.length is not None: + headers[b'Content-Length'] = str(response.length).encode() + for key, value in response.headers.getAllRawHeaders(): + headers[key] = value + return headers + def _cb_bodyready(self, txresponse, request): headers_received_result = self._crawler.signals.send_catch_log( signal=signals.headers_received, - headers=Headers(txresponse.headers.getAllRawHeaders()), + headers=self._headers_from_twisted_response(txresponse), body_length=txresponse.length, request=request, spider=self._crawler.spider, @@ -435,7 +444,7 @@ class ScrapyAgent: return d def _cb_bodydone(self, result, request, url): - headers = Headers(result["txresponse"].headers.getAllRawHeaders()) + headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) try: version = result["txresponse"].version diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 86d72772c..fa7d5c8a6 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -364,6 +364,13 @@ class HttpTestCase(unittest.TestCase): d.addCallback(self.assertEqual, body) return d + def test_response_header_content_length(self): + request = Request(self.getURL("file"), method=b"GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.headers[b'content-length']) + d.addCallback(self.assertEqual, b'159') + return d + class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" From 72e8cea8afb85f6704fcf6f5648b0a01f1abaab3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 22 Mar 2021 11:51:11 -0300 Subject: [PATCH 492/568] Avoid exceptions in is_generator_with_return_value (#4935) --- scrapy/utils/misc.py | 27 ++- ...t_return_with_argument_inside_generator.py | 167 +++++++++++++++--- 2 files changed, 161 insertions(+), 33 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 081cd33f1..5c986eedc 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -9,7 +9,6 @@ from collections import deque from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules -from textwrap import dedent from w3lib.html import replace_entities @@ -227,7 +226,8 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - tree = ast.parse(dedent(inspect.getsource(callable))) + code = re.sub(r"^[\t ]+", "", inspect.getsource(callable)) + tree = ast.parse(code) for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): _generator_callbacks_cache[callable] = True @@ -242,12 +242,23 @@ def warn_on_generator_with_return_value(spider, callable): Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None """ - if is_generator_with_return_value(callable): + try: + if is_generator_with_return_value(callable): + warnings.warn( + f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' + 'a generator and includes a "return" statement with a value ' + 'different than None. This could lead to unexpected behaviour. Please see ' + 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' + 'for details about the semantics of the "return" statement within generators', + stacklevel=2, + ) + except IndentationError: + callable_name = spider.__class__.__name__ + "." + callable.__name__ warnings.warn( - f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' - 'a generator and includes a "return" statement with a value ' - 'different than None. This could lead to unexpected behaviour. Please see ' - 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' - 'for details about the semantics of the "return" statement within generators', + f'Unable to determine whether or not "{callable_name}" is a generator with a return value. ' + 'This will not prevent your code from working, but it prevents Scrapy from detecting ' + f'potential issues in your implementation of "{callable_name}". Please, report this in the ' + 'Scrapy issue tracker (https://github.com/scrapy/scrapy/issues), ' + f'including the code of "{callable_name}"', stacklevel=2, ) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 2be38620c..1c85ca353 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,35 +1,116 @@ import unittest +import warnings +from unittest import mock -from scrapy.utils.misc import is_generator_with_return_value +from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value + + +def _indentation_error(*args, **kwargs): + raise IndentationError() + + +def top_level_return_something(): + """ +docstring + """ + url = """ +https://example.org +""" + yield url + return 1 + + +def top_level_return_none(): + """ +docstring + """ + url = """ +https://example.org +""" + yield url + return + + +def generator_that_returns_stuff(): + yield 1 + yield 2 + return 3 class UtilsMiscPy3TestCase(unittest.TestCase): - def test_generators_with_return_statements(self): - def f(): + def test_generators_return_something(self): + def f1(): yield 1 return 2 - def g(): + def g1(): yield 1 - return 'asdf' + return "asdf" - def h(): + def h1(): + yield 1 + + def helper(): + return 0 + + yield helper() + return 2 + + def i1(): + """ +docstring + """ + url = """ +https://example.org + """ + yield url + return 1 + + assert is_generator_with_return_value(top_level_return_something) + assert is_generator_with_return_value(f1) + assert is_generator_with_return_value(g1) + assert is_generator_with_return_value(h1) + assert is_generator_with_return_value(i1) + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_something) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.top_level_return_something" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, f1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.f1" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, g1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.g1" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, h1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.h1" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, i1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.i1" method is a generator', str(w[0].message)) + + def test_generators_return_none(self): + def f2(): yield 1 return None - def i(): + def g2(): yield 1 return - def j(): + def h2(): yield 1 - def k(): + def i2(): yield 1 - yield from g() + yield from generator_that_returns_stuff() - def m(): + def j2(): yield 1 def helper(): @@ -37,20 +118,56 @@ class UtilsMiscPy3TestCase(unittest.TestCase): yield helper() - def n(): - yield 1 + def k2(): + """ +docstring + """ + url = """ +https://example.org + """ + yield url + return - def helper(): - return 0 + def l2(): + return - yield helper() - return 2 + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f2) + assert not is_generator_with_return_value(g2) + assert not is_generator_with_return_value(h2) + assert not is_generator_with_return_value(i2) + assert not is_generator_with_return_value(j2) # not recursive + assert not is_generator_with_return_value(k2) # not recursive + assert not is_generator_with_return_value(l2) - assert is_generator_with_return_value(f) - assert is_generator_with_return_value(g) - assert not is_generator_with_return_value(h) - assert not is_generator_with_return_value(i) - assert not is_generator_with_return_value(j) - assert not is_generator_with_return_value(k) # not recursive - assert not is_generator_with_return_value(m) - assert is_generator_with_return_value(n) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_none) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, f2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, g2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, h2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, i2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, j2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, k2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, l2) + self.assertEqual(len(w), 0) + + @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error) + def test_indentation_error(self): + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_none) + self.assertEqual(len(w), 1) + self.assertIn('Unable to determine', str(w[0].message)) From 64d4ae1a19eda9c6bb0b6dc85e5dfff6187d0ca8 Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 22 Mar 2021 21:46:05 +0100 Subject: [PATCH 493/568] Update UsageError message --- scrapy/commands/crawl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index f205c40b0..0f2a21b85 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand): if len(args) < 1: raise UsageError() elif len(args) > 1: - raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") + raise UsageError("running 'scrapy crawl' with more than one spider is not supported") spname = args[0] crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) From f0e1a33225dd7c9162e277626df70284f41a427e Mon Sep 17 00:00:00 2001 From: Pratik Mahankal <53421565+pratik1500@users.noreply.github.com> Date: Tue, 23 Mar 2021 22:46:50 +0530 Subject: [PATCH 494/568] Sort the list of Request.meta alphabetically #5061 (#5065) --- docs/topics/request-response.rst | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a0448c5ab..c0283df01 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -363,26 +363,26 @@ are some special keys recognized by Scrapy and its built-in extensions. Those are: -* :reqmeta:`dont_redirect` -* :reqmeta:`dont_retry` -* :reqmeta:`handle_httpstatus_list` -* :reqmeta:`handle_httpstatus_all` -* :reqmeta:`dont_merge_cookies` +* :reqmeta:`bindaddress` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` +* :reqmeta:`dont_merge_cookies` +* :reqmeta:`dont_obey_robotstxt` +* :reqmeta:`dont_redirect` +* :reqmeta:`dont_retry` +* :reqmeta:`download_fail_on_dataloss` +* :reqmeta:`download_latency` +* :reqmeta:`download_maxsize` +* :reqmeta:`download_timeout` +* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) +* ``ftp_user`` (See :setting:`FTP_USER` for more info) +* :reqmeta:`handle_httpstatus_all` +* :reqmeta:`handle_httpstatus_list` +* :reqmeta:`max_retry_times` +* :reqmeta:`proxy` * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` -* :reqmeta:`bindaddress` -* :reqmeta:`dont_obey_robotstxt` -* :reqmeta:`download_timeout` -* :reqmeta:`download_maxsize` -* :reqmeta:`download_latency` -* :reqmeta:`download_fail_on_dataloss` -* :reqmeta:`proxy` -* ``ftp_user`` (See :setting:`FTP_USER` for more info) -* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * :reqmeta:`referrer_policy` -* :reqmeta:`max_retry_times` .. reqmeta:: bindaddress From 9c9e1a318d83b8e55125fdf7a7fb9482cb4951f8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 25 Mar 2021 11:58:39 -0300 Subject: [PATCH 495/568] [HTTP/1.1] Skip Content-Length header if its value is UNKNOWN_LENGTH (#5062) --- scrapy/core/downloader/handlers/http11.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1f82751fd..25cb3ec62 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -361,10 +361,9 @@ class ScrapyAgent: @staticmethod def _headers_from_twisted_response(response): headers = Headers() - if response.length is not None: + if response.length != UNKNOWN_LENGTH: headers[b'Content-Length'] = str(response.length).encode() - for key, value in response.headers.getAllRawHeaders(): - headers[key] = value + headers.update(response.headers.getAllRawHeaders()) return headers def _cb_bodyready(self, txresponse, request): From 1d200258a527668186ea84f14e7c2b23b3200ab5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Fri, 26 Mar 2021 10:45:26 -0300 Subject: [PATCH 496/568] Adjust h2 version requirement (#5066) --- setup.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 767c6f6bf..2b60a10af 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ install_requires = [ 'zope.interface>=4.1.3', 'protego>=0.1.15', 'itemadapter>=0.1.0', - 'h2>=3.2.0', + 'h2>=3.0,<4.0', ] extras_require = {} cpython_dependencies = [ diff --git a/tox.ini b/tox.ini index 6907c8906..353977519 100644 --- a/tox.ini +++ b/tox.ini @@ -71,7 +71,7 @@ commands = deps = cryptography==2.0 cssselect==0.9.1 - h2==3.2.0 + h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 From b247fa9982a390e1380c46e390069c6058d97921 Mon Sep 17 00:00:00 2001 From: Ricardo Amendoeira Date: Mon, 29 Mar 2021 01:48:28 +0100 Subject: [PATCH 497/568] Include loading settings in `Running multiple spiders in the same process` section The example in the documentation doesn't take into account the project settings --- docs/topics/practices.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cf1de1bd1..db1ed362e 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -118,6 +118,7 @@ Here is an example that runs multiple spiders simultaneously: :: import scrapy + from scrapy.utils.project import get_project_settings from scrapy.crawler import CrawlerProcess class MySpider1(scrapy.Spider): @@ -128,7 +129,8 @@ Here is an example that runs multiple spiders simultaneously: # Your second spider definition ... - process = CrawlerProcess() + settings = get_project_settings() + process = CrawlerProcess(settings) process.crawl(MySpider1) process.crawl(MySpider2) process.start() # the script will block here until all crawling jobs are finished From 90fe494ba2ab66faa49eb73914a1eae3cebacd0a Mon Sep 17 00:00:00 2001 From: Veniamin Gvozdikov Date: Thu, 1 Apr 2021 11:11:28 +0300 Subject: [PATCH 498/568] Rebranding, updated GA code --- docs/_templates/layout.html | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index a6f6cbda8..18a5231ee 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -3,14 +3,9 @@ {% block footer %} {{ super() }} {% endblock %} From d458ccff3b2d6df94df1aa86eeb7d2505d62f2d6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 1 Apr 2021 12:27:35 -0300 Subject: [PATCH 499/568] Retry request: priority_adjust cannot be float (Request.priority is int) --- scrapy/downloadermiddlewares/retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 2721db7cf..5965a1c6c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -41,7 +41,7 @@ def get_retry_request( spider: Spider, reason: Union[str, Exception] = 'unspecified', max_retry_times: Optional[int] = None, - priority_adjust: Union[int, float, None] = None, + priority_adjust: Optional[int] = None, logger: Logger = retry_logger, stats_base_key: str = 'retry', ): From 5492972d8a874e9fb7780175e4f396bc8f39378c Mon Sep 17 00:00:00 2001 From: anay2103 <55763427+anay2103@users.noreply.github.com> Date: Thu, 1 Apr 2021 20:30:48 +0300 Subject: [PATCH 500/568] added customized filtering examples in logging.rst (#4965) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added customized filtering examples in logging.rst * Update logging.rst * Update docs/topics/logging.rst Co-authored-by: Adrián Chaves * Update docs/topics/logging.rst Co-authored-by: Adrián Chaves * Update docs/topics/logging.rst Co-authored-by: Adrián Chaves * Update docs/topics/logging.rst Co-authored-by: Adrián Chaves * Update logging.rst Co-authored-by: Adrián Chaves --- docs/topics/logging.rst | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index c3445d40e..00806392a 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -242,6 +242,47 @@ e.g. in the spider's ``__init__`` method:: If you run this spider again then INFO messages from ``scrapy.spidermiddlewares.httperror`` logger will be gone. +You can also filter log records by :class:`~logging.LogRecord` data. For +example, you can filter log records by message content using a substring or +a regular expression. Create a :class:`logging.Filter` subclass +and equip it with a regular expression pattern to +filter out unwanted messages:: + + import logging + import re + + class ContentFilter(logging.Filter): + def filter(self, record): + match = re.search(r'\d{3} [Ee]rror, retrying', record.message) + if match: + return False + +A project-level filter may be attached to the root +handler created by Scrapy, this is a wieldy way to +filter all loggers in different parts of the project +(middlewares, spider, etc.):: + + import logging + import scrapy + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + for handler in logging.root.handlers: + handler.addFilter(ContentFilter()) + +Alternatively, you may choose a specific logger +and hide it without affecting other loggers:: + + import logging + import scrapy + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + logger = logging.getLogger('my_logger') + logger.addFilter(ContentFilter()) + scrapy.utils.log module ======================= From ad7456746961fb23beb3ebc051ee1ab0ed82a91e Mon Sep 17 00:00:00 2001 From: Kader DJEHAF Date: Thu, 1 Apr 2021 19:33:56 +0200 Subject: [PATCH 501/568] Fix argument type (int -> bool) (#4950) * Fix warning: Expected type 'bool', got 'int' instead * Update defer.py * Fix warning: Expected type 'bool', got 'int' instead Co-authored-by: Eugenio Lacuesta --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 7f267c800..ef2eb3ba6 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -71,7 +71,7 @@ def _urlencode(seq, enc): values = [(to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq for v in (vs if is_listlike(vs) else [vs])] - return urlencode(values, doseq=1) + return urlencode(values, doseq=True) def _get_form(response, formname, formid, formnumber, formxpath): From cc095aa8950975cf203b1d33724cc499043e3f17 Mon Sep 17 00:00:00 2001 From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com> Date: Thu, 1 Apr 2021 23:09:33 +0530 Subject: [PATCH 502/568] Windows pip installation guide (#4736) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added initial steps * fixing link * python3 -> python * remaining steps * steps updated * Update docs/intro/install.rst Co-authored-by: Adrián Chaves * added link to Visual Studio * removed 'install V' * Update docs/intro/install.rst Co-authored-by: Adrián Chaves Co-authored-by: Adrián Chaves --- docs/intro/install.rst | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index bf919ce25..8581dde0b 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -12,6 +12,7 @@ Supported Python versions Scrapy requires Python 3.6+, either the CPython implementation (default) or the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). +.. _intro-install-scrapy: Installing Scrapy ================= @@ -29,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with:: pip install Scrapy +We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, +to avoid conflicting with your system packages. + Note that sometimes this may require solving compilation issues for some Scrapy dependencies depending on your operating system, so be sure to check the :ref:`intro-install-platform-notes`. -We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, -to avoid conflicting with your system packages. - For more detailed and platform specifics instructions, as well as troubleshooting information, read on. @@ -117,6 +118,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: conda install -c conda-forge scrapy +To install Scrapy on Windows using ``pip``: + +.. warning:: + This installation method requires “Microsoft Visual C++” for installing some + Scrapy dependencies, which demands significantly more disk space than Anaconda. + +#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. + +#. Run the Visual Studio Installer. + +#. Under the Workloads section, select **C++ build tools**. + +#. Check the installation details and make sure following packages are selected as optional components: + + * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) + + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) + +#. Install the Visual Studio Build Tools. + +Now, you should be able to :ref:`install Scrapy ` using ``pip``. .. _intro-install-ubuntu: @@ -268,4 +290,6 @@ For details, see `Issue #2473 `_. .. _zsh: https://www.zsh.org/ .. _Anaconda: https://docs.anaconda.com/anaconda/ .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html +.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio +.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/ .. _conda-forge: https://conda-forge.org/ From 9e7cbc05ae10ab6c1335d92b4d41429ba8e2bf24 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 1 Apr 2021 15:22:51 -0300 Subject: [PATCH 503/568] Fix type for urlencode's doseq argument --- scrapy/commands/bench.py | 2 +- tests/spiders.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 999c987ea..6bdf9eae0 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider): def start_requests(self): qargs = {'total': self.total, 'show': self.show} - url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}' + url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}' return [scrapy.Request(url, dont_filter=True)] def parse(self, response): diff --git a/tests/spiders.py b/tests/spiders.py index 7e579098a..5b45f897e 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -45,7 +45,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") self.start_urls = [url] def parse(self, response): @@ -245,7 +245,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 From 9e3b868dd83a3ac93dfe04b53d5130145154f478 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 3 Apr 2021 17:04:09 +0500 Subject: [PATCH 504/568] Use __qualname__ in middleware handling. --- scrapy/core/downloader/middleware.py | 9 +++------ scrapy/core/spidermw.py | 10 +++------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index b0e612e43..177f3f760 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -36,8 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_request must return None, Response or " + f"Middleware {method.__qualname__} must return None, Response or " f"Request, got {response.__class__.__name__}" ) if response: @@ -55,8 +54,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_response must return Response or Request, " + f"Middleware {method.__qualname__} must return Response or Request, " f"got {type(response)}" ) if isinstance(response, Request): @@ -70,8 +68,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_exception must return None, Response or " + f"Middleware {method.__qualname__} must return None, Response or " f"Request, got {type(response)}" ) if response: diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 289292da7..e8733c4ad 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -18,10 +18,6 @@ def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') -def _fname(f): - return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" - - class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -46,7 +42,7 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - msg = (f"Middleware {_fname(method)} must return None " + msg = (f"Middleware {method.__qualname__} must return None " f"or raise an exception, got {type(result)}") raise _InvalidOutput(msg) except _InvalidOutput: @@ -83,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = (f"Middleware {_fname(method)} must return None " + msg = (f"Middleware {method.__qualname__} must return None " f"or an iterable, got {type(result)}") raise _InvalidOutput(msg) return _failure @@ -108,7 +104,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) else: - msg = (f"Middleware {_fname(method)} must return an " + msg = (f"Middleware {method.__qualname__} must return an " f"iterable, got {type(result)}") raise _InvalidOutput(msg) From a9e96f99077865bc2d844ecf0ae174db97ba6d8b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 3 Apr 2021 17:40:45 +0500 Subject: [PATCH 505/568] Add typing for middleware and coroutine related code. --- scrapy/core/downloader/middleware.py | 14 ++++++++----- scrapy/core/spidermw.py | 31 ++++++++++++++++++---------- scrapy/middleware.py | 11 +++++----- scrapy/utils/asyncgen.py | 5 ++++- scrapy/utils/defer.py | 23 ++++++++++++--------- scrapy/utils/python.py | 7 ++++--- 6 files changed, 56 insertions(+), 35 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index b0e612e43..441fc9fa6 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,8 +3,12 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from twisted.internet import defer +from typing import Callable, Union +from twisted.internet import defer +from twisted.python.failure import Failure + +from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager @@ -29,9 +33,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process_exception'): self.methods['process_exception'].appendleft(mw.process_exception) - def download(self, download_func, request, spider): + def download(self, download_func: Callable, request: Request, spider: Spider): @defer.inlineCallbacks - def process_request(request): + def process_request(request: Request): for method in self.methods['process_request']: response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): @@ -45,7 +49,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return (yield download_func(request=request, spider=spider)) @defer.inlineCallbacks - def process_response(response): + def process_response(response: Union[Response, Request]): if response is None: raise TypeError("Received None in process_response") elif isinstance(response, Request): @@ -64,7 +68,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response @defer.inlineCallbacks - def process_exception(failure): + def process_exception(failure: Failure): exception = failure.value for method in self.methods['process_exception']: response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 289292da7..b09adf8e2 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,25 +3,32 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +from collections.abc import Iterable, AsyncIterable from itertools import islice +from typing import Callable, Union, Any from twisted.python.failure import Failure +from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput +from scrapy.http import Response from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list from scrapy.utils.defer import mustbe_deferred from scrapy.utils.python import MutableChain -def _isiterable(possible_iterator): - return hasattr(possible_iterator, '__iter__') +def _isiterable(o): + return isinstance(o, Iterable) def _fname(f): return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" +ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] + + class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -41,7 +48,7 @@ class SpiderMiddlewareManager(MiddlewareManager): process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) - def _process_spider_input(self, scrape_func, response, request, spider): + def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider): for method in self.methods['process_spider_input']: try: result = method(response=response, spider=spider) @@ -55,7 +62,8 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to): + def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable, + exception_processor_index: int, recover_to: MutableChain): try: for r in iterable: yield r @@ -66,7 +74,7 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recover_to.extend(exception_result) - def _process_spider_exception(self, response, spider, _failure, start_index=0): + def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index=0): exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -88,7 +96,8 @@ class SpiderMiddlewareManager(MiddlewareManager): raise _InvalidOutput(msg) return _failure - def _process_spider_output(self, response, spider, result, start_index=0): + def _process_spider_output(self, response: Response, spider: Spider, + result: Iterable, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered = MutableChain() @@ -114,21 +123,21 @@ class SpiderMiddlewareManager(MiddlewareManager): return MutableChain(result, recovered) - def _process_callback_output(self, response, spider, result): + def _process_callback_output(self, response: Response, spider: Spider, result: Iterable): recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) return MutableChain(self._process_spider_output(response, spider, result), recovered) - def scrape_response(self, scrape_func, response, request, spider): - def process_callback_output(result): + def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider): + def process_callback_output(result: Iterable): return self._process_callback_output(response, spider, result) - def process_spider_exception(_failure): + def process_spider_exception(_failure: Failure): return self._process_spider_exception(response, spider, _failure) dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) return dfd - def process_start_requests(self, start_requests, spider): + def process_start_requests(self, start_requests, spider: Spider): return self._process_chain('process_start_requests', start_requests, spider) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 5040378ea..c53cfb814 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,6 +1,7 @@ -from collections import defaultdict, deque import logging import pprint +from collections import defaultdict, deque +from typing import Callable from scrapy.exceptions import NotConfigured from scrapy.utils.misc import create_instance, load_object @@ -16,7 +17,7 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods = defaultdict(deque) + self.methods: dict[str, deque[Callable]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -58,13 +59,13 @@ class MiddlewareManager: if hasattr(mw, 'close_spider'): self.methods['close_spider'].appendleft(mw.close_spider) - def _process_parallel(self, methodname, obj, *args): + def _process_parallel(self, methodname: str, obj, *args): return process_parallel(self.methods[methodname], obj, *args) - def _process_chain(self, methodname, obj, *args): + def _process_chain(self, methodname: str, obj, *args): return process_chain(self.methods[methodname], obj, *args) - def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args): + def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args): return process_chain_both(self.methods[cb_methodname], self.methods[eb_methodname], obj, *args) diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 7f697af5f..c290e376c 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,4 +1,7 @@ -async def collect_asyncgen(result): +from collections.abc import AsyncIterable + + +async def collect_asyncgen(result: AsyncIterable): results = [] async for x in result: results.append(x) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 6db9cc117..c382a00f7 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -3,16 +3,19 @@ Helper functions for dealing with Twisted deferreds """ import asyncio import inspect +from collections.abc import Coroutine from functools import wraps +from typing import Callable, Iterable, Any from twisted.internet import defer, task from twisted.python import failure +from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import is_asyncio_reactor_installed -def defer_fail(_failure): +def defer_fail(_failure: Failure): """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -47,7 +50,7 @@ def defer_result(result): return defer_succeed(result) -def mustbe_deferred(f, *args, **kw): +def mustbe_deferred(f: Callable, *args, **kw): """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -64,7 +67,7 @@ def mustbe_deferred(f, *args, **kw): return defer_result(result) -def parallel(iterable, count, callable, *args, **named): +def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named): """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -75,7 +78,7 @@ def parallel(iterable, count, callable, *args, **named): return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) -def process_chain(callbacks, input, *a, **kw): +def process_chain(callbacks: Iterable[Callable], input, *a, **kw): """Return a Deferred built by chaining the given callbacks""" d = defer.Deferred() for x in callbacks: @@ -84,7 +87,7 @@ def process_chain(callbacks, input, *a, **kw): return d -def process_chain_both(callbacks, errbacks, input, *a, **kw): +def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw): """Return a Deferred built by chaining the given callbacks and errbacks""" d = defer.Deferred() for cb, eb in zip(callbacks, errbacks): @@ -100,7 +103,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw): return d -def process_parallel(callbacks, input, *a, **kw): +def process_parallel(callbacks: Iterable[Callable], input, *a, **kw): """Return a Deferred with the output of all successful calls to the given callbacks """ @@ -110,7 +113,7 @@ def process_parallel(callbacks, input, *a, **kw): return d -def iter_errback(iterable, errback, *a, **kw): +def iter_errback(iterable: Iterable, errback: Callable, *a, **kw): """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -124,7 +127,7 @@ def iter_errback(iterable, errback, *a, **kw): errback(failure.Failure(), *a, **kw) -def deferred_from_coro(o): +def deferred_from_coro(o) -> Any: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, defer.Deferred): return o @@ -139,7 +142,7 @@ def deferred_from_coro(o): return o -def deferred_f_from_coro_f(coro_f): +def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]): """ Converts a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. @@ -151,7 +154,7 @@ def deferred_f_from_coro_f(coro_f): return f -def maybeDeferred_coro(f, *args, **kw): +def maybeDeferred_coro(f: Callable, *args, **kw): """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ try: result = f(*args, **kw) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5703fd4c3..bcc12f24f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,6 +8,7 @@ import re import sys import warnings import weakref +from collections.abc import Iterable from functools import partial, wraps from itertools import chain @@ -335,15 +336,15 @@ else: gc.collect() -class MutableChain: +class MutableChain(Iterable): """ Thin wrapper around itertools.chain, allowing to add iterables "in-place" """ - def __init__(self, *args): + def __init__(self, *args: Iterable): self.data = chain.from_iterable(args) - def extend(self, *iterables): + def extend(self, *iterables: Iterable): self.data = chain(self.data, chain.from_iterable(iterables)) def __iter__(self): From 414dd1119a7e15c612de7bd0fe560b6ca30b505f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 3 Apr 2021 17:54:55 +0500 Subject: [PATCH 506/568] Drop an unused import. --- scrapy/core/spidermw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index b09adf8e2..9a5305376 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,7 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -from collections.abc import Iterable, AsyncIterable +from collections.abc import Iterable from itertools import islice from typing import Callable, Union, Any From 7dc857668f16e8c52ff44662aceb32f93ae3d80e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 4 Apr 2021 16:15:33 +0500 Subject: [PATCH 507/568] Also some typing for Scraper. --- scrapy/core/scraper.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 0d3e3450f..4a3eff888 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -3,12 +3,14 @@ extracts information from them""" import logging from collections import deque +from collections.abc import Iterable +from typing import Union from itemadapter import is_item from twisted.internet import defer from twisted.python.failure import Failure -from scrapy import signals +from scrapy import signals, Spider from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response @@ -120,7 +122,7 @@ class Scraper: response, request, deferred = slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, result, request, spider): + def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider): """ Handle the downloaded response or failure through the spider callback/errback """ @@ -131,7 +133,7 @@ class Scraper: dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, result, request, spider): + def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider): """ Handle the different cases of request's result been a Response or a Failure """ @@ -141,7 +143,7 @@ class Scraper: dfd = self.call_spider(result, request, spider) return dfd.addErrback(self._log_download_errors, result, request, spider) - def call_spider(self, result, request, spider): + def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider): if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request @@ -156,7 +158,7 @@ class Scraper: dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) - def handle_spider_error(self, _failure, request, response, spider): + def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider): exc = _failure.value if isinstance(exc, CloseSpider): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') @@ -177,7 +179,7 @@ class Scraper: spider=spider ) - def handle_spider_output(self, result, request, response, spider): + def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider): if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, response, spider) From e7d51886ef90f5b2d4fa13911381680ac192fc37 Mon Sep 17 00:00:00 2001 From: Mayank Singhal <17mayanksinghal@gmail.com> Date: Tue, 6 Apr 2021 02:21:18 +0530 Subject: [PATCH 508/568] Find bash from PATH instead of /bin/bash --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index ff68bf1ae..87d5d3047 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,7 +8,7 @@ PYTHON = python SPHINXOPTS = PAPER = SOURCES = -SHELL = /bin/bash +SHELL = /usr/bin/env bash ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ -D latex_elements.papersize=$(PAPER) \ From a71d6ef29da8e3dfa906fbaa74ee1db64567f856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 6 Apr 2021 16:09:07 +0200 Subject: [PATCH 509/568] 2.5.0 release notes (#5028) Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> --- docs/news.rst | 193 ++++++++++++++++++++++++++++++- docs/topics/request-response.rst | 4 +- docs/topics/settings.rst | 5 +- docs/topics/signals.rst | 2 +- tests/test_webclient.py | 2 +- 5 files changed, 200 insertions(+), 6 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index d9fe897ad..0ea412e75 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,190 @@ Release notes ============= +.. _release-2.5.0: + +Scrapy 2.5.0 (2021-04-06) +------------------------- + +Highlights: + +- Official Python 3.9 support + +- Experimental :ref:`HTTP/2 support ` + +- New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function + to retry requests from spider callbacks + +- New :class:`~scrapy.signals.headers_received` signal that allows stopping + downloads early + +- New :class:`Response.protocol ` attribute + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed all code that :ref:`was deprecated in 1.7.0 <1.7-deprecations>` and + had not :ref:`already been removed in 2.4.0 <2.4-deprecation-removals>`. + (:issue:`4901`) + +- Removed support for the ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment + variable, :ref:`deprecated in 1.8.0 <1.8-deprecations>`. (:issue:`4912`) + + +Deprecations +~~~~~~~~~~~~ + +- The :mod:`scrapy.utils.py36` module is now deprecated in favor of + :mod:`scrapy.utils.asyncgen`. (:issue:`4900`) + + +New features +~~~~~~~~~~~~ + +- Experimental :ref:`HTTP/2 support ` through a new download handler + that can be assigned to the ``https`` protocol in the + :setting:`DOWNLOAD_HANDLERS` setting. + (:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`) + +- The new :func:`scrapy.downloadermiddlewares.retry.get_retry_request` + function may be used from spider callbacks or middlewares to handle the + retrying of a request beyond the scenarios that + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` supports. + (:issue:`3590`, :issue:`3685`, :issue:`4902`) + +- The new :class:`~scrapy.signals.headers_received` signal gives early access + to response headers and allows :ref:`stopping downloads + `. + (:issue:`1772`, :issue:`4897`) + +- The new :attr:`Response.protocol ` + attribute gives access to the string that identifies the protocol used to + download a response. (:issue:`4878`) + +- :ref:`Stats ` now include the following entries that indicate + the number of successes and failures in storing + :ref:`feeds `:: + + feedexport/success_count/ + feedexport/failed_count/ + + Where ```` is the feed storage backend class name, such as + :class:`~scrapy.extensions.feedexport.FileFeedStorage` or + :class:`~scrapy.extensions.feedexport.FTPFeedStorage`. + + (:issue:`3947`, :issue:`4850`) + +- The :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` spider + middleware now logs ignored URLs with ``INFO`` :ref:`logging level + ` instead of ``DEBUG``, and it now includes the following entry + into :ref:`stats ` to keep track of the number of ignored + URLs:: + + urllength/request_ignored_count + + (:issue:`5036`) + +- The + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + downloader middleware now logs the number of decompressed responses and the + total count of resulting bytes:: + + httpcompression/response_bytes + httpcompression/response_count + + (:issue:`4797`, :issue:`4799`) + + +Bug fixes +~~~~~~~~~ + +- Fixed installation on PyPy installing PyDispatcher in addition to + PyPyDispatcher, which could prevent Scrapy from working depending on which + package got imported. (:issue:`4710`, :issue:`4814`) + +- When inspecting a callback to check if it is a generator that also returns + a value, an exception is no longer raised if the callback has a docstring + with lower indentation than the following code. + (:issue:`4477`, :issue:`4935`) + +- The `Content-Length `_ + header is no longer omitted from responses when using the default, HTTP/1.1 + download handler (see :setting:`DOWNLOAD_HANDLERS`). + (:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`) + +- Setting the :reqmeta:`handle_httpstatus_all` request meta key to ``False`` + now has the same effect as not setting it at all, instead of having the + same effect as setting it to ``True``. + (:issue:`3851`, :issue:`4694`) + + +Documentation +~~~~~~~~~~~~~ + +- Added instructions to :ref:`install Scrapy in Windows using pip + `. + (:issue:`4715`, :issue:`4736`) + +- Logging documentation now includes :ref:`additional ways to filter logs + `. + (:issue:`4216`, :issue:`4257`, :issue:`4965`) + +- Covered how to deal with long lists of allowed domains in the :ref:`FAQ + `. (:issue:`2263`, :issue:`3667`) + +- Covered scrapy-bench_ in :ref:`benchmarking`. + (:issue:`4996`, :issue:`5016`) + +- Clarified that one :ref:`extension ` instance is created + per crawler. + (:issue:`5014`) + +- Fixed some errors in examples. + (:issue:`4829`, :issue:`4830`, :issue:`4907`, :issue:`4909`, + :issue:`5008`) + +- Fixed some external links, typos, and so on. + (:issue:`4892`, :issue:`4899`, :issue:`4936`, :issue:`4942`, :issue:`5005`, + :issue:`5063`) + +- The :ref:`list of Request.meta keys ` is now sorted + alphabetically. + (:issue:`5061`, :issue:`5065`) + +- Updated references to Scrapinghub, which is now called Zyte. + (:issue:`4973`, :issue:`5072`) + +- Added a mention to contributors in the README. (:issue:`4956`) + +- Reduced the top margin of lists. (:issue:`4974`) + + +Quality Assurance +~~~~~~~~~~~~~~~~~ + +- Made Python 3.9 support official (:issue:`4757`, :issue:`4759`) + +- Extended typing hints (:issue:`4895`) + +- Fixed deprecated uses of the Twisted API. + (:issue:`4940`, :issue:`4950`, :issue:`5073`) + +- Made our tests run with the new pip resolver. + (:issue:`4710`, :issue:`4814`) + +- Added tests to ensure that :ref:`coroutine support ` + is tested. (:issue:`4987`) + +- Migrated from Travis CI to GitHub Actions. (:issue:`4924`) + +- Fixed CI issues. + (:issue:`4986`, :issue:`5020`, :issue:`5022`, :issue:`5027`, :issue:`5052`, + :issue:`5053`) + +- Implemented code refactorings, style fixes and cleanups. + (:issue:`4911`, :issue:`4982`, :issue:`5001`, :issue:`5002`, :issue:`5076`) + + .. _release-2.4.1: Scrapy 2.4.1 (2020-11-17) @@ -97,6 +281,8 @@ Backward-incompatible changes (:issue:`4717`, :issue:`4823`) +.. _2.4-deprecation-removals: + Deprecation removals ~~~~~~~~~~~~~~~~~~~~ @@ -1433,6 +1619,8 @@ Deprecation removals * ``scrapy.xlib`` has been removed (:issue:`4015`) +.. _1.8-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -1789,6 +1977,8 @@ The following deprecated settings have also been removed (:issue:`3578`): * ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`) +.. _1.7-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -4184,7 +4374,7 @@ API changes - ``url`` and ``body`` attributes of Request objects are now read-only (#230) - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231) - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default) -- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) +- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) - Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself. - Changes to Scrapy Manager (now called "Crawler"): - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` @@ -4331,6 +4521,7 @@ First release of Scrapy. .. _resource: https://docs.python.org/2/library/resource.html .. _robots.txt: https://www.robotstxt.org/ .. _scrapely: https://github.com/scrapy/scrapely +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench .. _service_identity: https://service-identity.readthedocs.io/en/stable/ .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index c0283df01..500781c05 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -703,7 +703,7 @@ Response objects .. versionadded:: 2.1.0 The ``ip_address`` parameter. - .. versionadded:: VERSION + .. versionadded:: 2.5.0 The ``protocol`` parameter. .. attribute:: Response.url @@ -809,7 +809,7 @@ Response objects .. attribute:: Response.protocol - .. versionadded:: VERSION + .. versionadded:: 2.5.0 The protocol that was used to download the response. For instance: "HTTP/1.0", "HTTP/1.1" diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 9dcee64eb..f5dca824f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -677,6 +677,8 @@ handler (without replacement), place this in your ``settings.py``:: 'ftp': None, } +.. _http2: + The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update :setting:`DOWNLOAD_HANDLERS` as follows:: @@ -703,7 +705,8 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update - No support for `server pushes`_, which are ignored. - - No support for the :signal:`bytes_received` signal. + - No support for the :signal:`bytes_received` and + :signal:`headers_received` signals. .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 98cfa606c..3d838fb63 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -403,7 +403,7 @@ bytes_received headers_received ~~~~~~~~~~~~~~~~ -.. versionadded:: VERSION +.. versionadded:: 2.5 .. signal:: headers_received .. function:: headers_received(headers, request, spider) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index f935a8689..6e4cb9b6e 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -418,7 +418,7 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): def testPayloadDisabledCipher(self): if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"): - self.skipTest("This does work in PyPy with cryptography<=2.3.1") + self.skipTest("This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1") s = "0123456789" * 10 settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) From e63188cbf753d560e43d8489c821bd6eb9fe54e9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 6 Apr 2021 19:13:32 +0500 Subject: [PATCH 510/568] =?UTF-8?q?Bump=20version:=202.4.1=20=E2=86=92=202?= =?UTF-8?q?.5.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 956c512cb..d9e4a2831 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.1 +current_version = 2.5.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 005119baa..437459cd9 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.4.1 +2.5.0 From 8603f9d7a5524b7709b4e8a8fc04a75a9a4f0ffe Mon Sep 17 00:00:00 2001 From: Ricardo Amendoeira Date: Tue, 6 Apr 2021 20:23:07 +0100 Subject: [PATCH 511/568] Apply changes to other examples in the same section. --- docs/topics/practices.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index db1ed362e..15ac520e2 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -118,8 +118,8 @@ Here is an example that runs multiple spiders simultaneously: :: import scrapy - from scrapy.utils.project import get_project_settings from scrapy.crawler import CrawlerProcess + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -143,6 +143,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: from twisted.internet import reactor from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -153,7 +154,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: ... configure_logging() - runner = CrawlerRunner() + settings = get_project_settings() + runner = CrawlerRunner(settings) runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() @@ -168,6 +170,7 @@ Same example but running the spiders sequentially by chaining the deferreds: from twisted.internet import reactor, defer from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -178,7 +181,8 @@ Same example but running the spiders sequentially by chaining the deferreds: ... configure_logging() - runner = CrawlerRunner() + settings = get_project_settings() + runner = CrawlerRunner(settings) @defer.inlineCallbacks def crawl(): From 5a75b14a5fbbbd37c14aa7317761655ac7706b70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 7 Apr 2021 12:33:37 +0200 Subject: [PATCH 512/568] docs: require sphinx-rtd-theme>=0.5.2 and the latest pip to prevent installing breaking docutils>=0.17 --- .readthedocs.yml | 1 - docs/pip.txt | 3 --- docs/requirements.txt | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 docs/pip.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index 2d781ae81..80a1cd036 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,6 +12,5 @@ python: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image version: 3.8 # Keep in sync with .github/workflows/checks.yml install: - - requirements: docs/pip.txt - requirements: docs/requirements.txt - path: . diff --git a/docs/pip.txt b/docs/pip.txt deleted file mode 100644 index 095e53a0d..000000000 --- a/docs/pip.txt +++ /dev/null @@ -1,3 +0,0 @@ -# In pip 20.3-21.0, the default dependency resolver causes the build in -# ReadTheDocs to fail due to memory exhaustion or timeout. -pip<20.3 diff --git a/docs/requirements.txt b/docs/requirements.txt index 3d34b47da..a0930ba1e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ Sphinx>=3.0 sphinx-hoverxref>=0.2b1 sphinx-notfound-page>=0.4 -sphinx_rtd_theme>=0.4 +sphinx-rtd-theme>=0.5.2 \ No newline at end of file From b6f77806b0ec414a28cfcbac3fa2d928040548d6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 9 Apr 2021 12:19:30 -0300 Subject: [PATCH 513/568] Engine tests: fix item class spider, add minimal type hints --- setup.cfg | 3 --- tests/test_engine.py | 24 +++++++++++------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/setup.cfg b/setup.cfg index 89b4ec57e..1fab6fe22 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,9 +43,6 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_engine] -ignore_errors = True - [mypy-tests.test_exporters] ignore_errors = True diff --git a/tests/test_engine.py b/tests/test_engine.py index ef1204f94..c406d2577 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -56,7 +56,7 @@ class TestSpider(Spider): name_re = re.compile(r"

    (.*?)

    ", re.M) price_re = re.compile(r">Price: \$(.*?)<", re.M) - item_cls = TestItem + item_cls: type = TestItem def parse(self, response): xlink = LinkExtractor() @@ -66,15 +66,15 @@ class TestSpider(Spider): yield Request(url=link.url, callback=self.parse_item) def parse_item(self, response): - item = self.item_cls() + adapter = ItemAdapter(self.item_cls()) m = self.name_re.search(response.text) if m: - item['name'] = m.group(1) - item['url'] = response.url + adapter['name'] = m.group(1) + adapter['url'] = response.url m = self.price_re.search(response.text) if m: - item['price'] = m.group(1) - return item + adapter['price'] = m.group(1) + return adapter.item class TestDupeFilterSpider(TestSpider): @@ -87,7 +87,7 @@ class DictItemsSpider(TestSpider): class AttrsItemsSpider(TestSpider): - item_class = AttrsItem + item_cls = AttrsItem try: @@ -97,14 +97,12 @@ except ImportError: else: TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) - class DataClassItemsSpider(DictItemsSpider): + class _dataclass_spider(DictItemsSpider): def parse_item(self, response): item = super().parse_item(response) - return TestDataClass( - name=item.get('name'), - url=item.get('url'), - price=item.get('price'), - ) + return TestDataClass(**item) + + DataClassItemsSpider = _dataclass_spider class ItemZeroDivisionErrorSpider(TestSpider): From 4673f05ddec85f18a890e606a6b85282035e26be Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 9 Apr 2021 23:42:24 +0500 Subject: [PATCH 514/568] Cleanup of slot handling in Scraper. --- scrapy/core/scraper.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 0d3e3450f..4a3ddea10 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -82,28 +82,26 @@ class Scraper: def close_spider(self, spider): """Close a spider being scraped and release its resources""" - slot = self.slot - slot.closing = defer.Deferred() - slot.closing.addCallback(self.itemproc.close_spider) - self._check_if_closing(spider, slot) - return slot.closing + self.slot.closing = defer.Deferred() + self.slot.closing.addCallback(self.itemproc.close_spider) + self._check_if_closing(spider) + return self.slot.closing def is_idle(self): """Return True if there isn't any more spiders to process""" return not self.slot - def _check_if_closing(self, spider, slot): - if slot.closing and slot.is_idle(): - slot.closing.callback(spider) + def _check_if_closing(self, spider): + if self.slot.closing and self.slot.is_idle(): + self.slot.closing.callback(spider) def enqueue_scrape(self, response, request, spider): - slot = self.slot - dfd = slot.add_response_request(response, request) + dfd = self.slot.add_response_request(response, request) def finish_scraping(_): - slot.finish_response(response, request) - self._check_if_closing(spider, slot) - self._scrape_next(spider, slot) + self.slot.finish_response(response, request) + self._check_if_closing(spider) + self._scrape_next(spider) return _ dfd.addBoth(finish_scraping) @@ -112,12 +110,12 @@ class Scraper: {'request': request}, exc_info=failure_to_exc_info(f), extra={'spider': spider})) - self._scrape_next(spider, slot) + self._scrape_next(spider) return dfd - def _scrape_next(self, spider, slot): - while slot.queue: - response, request, deferred = slot.next_response_request_deferred() + def _scrape_next(self, spider): + while self.slot.queue: + response, request, deferred = self.slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) def _scrape(self, result, request, spider): From d8d1dc5b508832598591ae9d3d3694f77f70492d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 12 Apr 2021 10:43:02 -0300 Subject: [PATCH 515/568] Ignore typing warning in test --- tests/test_engine.py | 4 +--- tox.ini | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index c406d2577..b2d1d83c7 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -97,13 +97,11 @@ except ImportError: else: TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) - class _dataclass_spider(DictItemsSpider): + class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef] def parse_item(self, response): item = super().parse_item(response) return TestDataClass(**item) - DataClassItemsSpider = _dataclass_spider - class ItemZeroDivisionErrorSpider(TestSpider): custom_settings = { diff --git a/tox.ini b/tox.ini index 353977519..5b0606f8f 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,7 @@ basepython = python3 deps = mypy==0.780 commands = - mypy {posargs: scrapy tests} + mypy --show-error-codes {posargs: scrapy tests} [testenv:security] basepython = python3 From a4415e4e6fa0f9becb40968438a77f4ea262633a Mon Sep 17 00:00:00 2001 From: Mayank Singhal <17mayanksinghal@gmail.com> Date: Tue, 13 Apr 2021 17:20:55 +0530 Subject: [PATCH 516/568] Add DataURI download handler in DOWNLOAD_HANDLERS_BASE documentation --- docs/topics/settings.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f5dca824f..1d5babcec 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -657,6 +657,7 @@ DOWNLOAD_HANDLERS_BASE Default:: { + 'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler', 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', From 76fa2257ef0280fc82e123457c791254cc2f185e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Apr 2021 20:01:18 +0500 Subject: [PATCH 517/568] Add typing also for return values, other small fixes. --- scrapy/core/spidermw.py | 33 ++++++++++++++++++--------------- scrapy/middleware.py | 24 ++++++++++++++---------- scrapy/utils/defer.py | 24 ++++++++++++------------ 3 files changed, 44 insertions(+), 37 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index dc0e58095..05df8c988 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,10 +3,10 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -from collections.abc import Iterable from itertools import islice -from typing import Callable, Union, Any +from typing import Callable, Union, Any, Generator, Iterable +from twisted.internet.defer import Deferred from twisted.python.failure import Failure from scrapy import Request, Spider @@ -18,13 +18,13 @@ from scrapy.utils.defer import mustbe_deferred from scrapy.utils.python import MutableChain -def _isiterable(o): - return isinstance(o, Iterable) - - ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] +def _isiterable(o) -> bool: + return isinstance(o, Iterable) + + class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -44,7 +44,8 @@ class SpiderMiddlewareManager(MiddlewareManager): process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) - def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider): + def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, + spider: Spider) -> Any: for method in self.methods['process_spider_input']: try: result = method(response=response, spider=spider) @@ -59,7 +60,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(response, request, spider) def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable, - exception_processor_index: int, recover_to: MutableChain): + exception_processor_index: int, recover_to: MutableChain) -> Generator: try: for r in iterable: yield r @@ -70,7 +71,8 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recover_to.extend(exception_result) - def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index=0): + def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, + start_index: int = 0) -> Union[Failure, MutableChain]: exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -93,7 +95,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return _failure def _process_spider_output(self, response: Response, spider: Spider, - result: Iterable, start_index=0): + result: Iterable, start_index: int = 0) -> MutableChain: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered = MutableChain() @@ -119,21 +121,22 @@ class SpiderMiddlewareManager(MiddlewareManager): return MutableChain(result, recovered) - def _process_callback_output(self, response: Response, spider: Spider, result: Iterable): + def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) return MutableChain(self._process_spider_output(response, spider, result), recovered) - def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider): - def process_callback_output(result: Iterable): + def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, + spider: Spider) -> Deferred: + def process_callback_output(result: Iterable) -> MutableChain: return self._process_callback_output(response, spider, result) - def process_spider_exception(_failure: Failure): + def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: return self._process_spider_exception(response, spider, _failure) dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) return dfd - def process_start_requests(self, start_requests, spider: Spider): + def process_start_requests(self, start_requests, spider: Spider) -> Deferred: return self._process_chain('process_start_requests', start_requests, spider) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index c53cfb814..3f8c1cbf5 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,9 +1,13 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable +from typing import Callable, Dict, Deque +from twisted.internet import defer + +from scrapy import Spider from scrapy.exceptions import NotConfigured +from scrapy.settings import Settings from scrapy.utils.misc import create_instance, load_object from scrapy.utils.defer import process_parallel, process_chain, process_chain_both @@ -17,16 +21,16 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods: dict[str, deque[Callable]] = defaultdict(deque) + self.methods: Dict[str, Deque[Callable]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: Settings) -> list: raise NotImplementedError @classmethod - def from_settings(cls, settings, crawler=None): + def from_settings(cls, settings: Settings, crawler=None): mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] @@ -53,24 +57,24 @@ class MiddlewareManager: def from_crawler(cls, crawler): return cls.from_settings(crawler.settings, crawler) - def _add_middleware(self, mw): + def _add_middleware(self, mw) -> None: if hasattr(mw, 'open_spider'): self.methods['open_spider'].append(mw.open_spider) if hasattr(mw, 'close_spider'): self.methods['close_spider'].appendleft(mw.close_spider) - def _process_parallel(self, methodname: str, obj, *args): + def _process_parallel(self, methodname: str, obj, *args) -> defer.Deferred: return process_parallel(self.methods[methodname], obj, *args) - def _process_chain(self, methodname: str, obj, *args): + def _process_chain(self, methodname: str, obj, *args) -> defer.Deferred: return process_chain(self.methods[methodname], obj, *args) - def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args): + def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> defer.Deferred: return process_chain_both(self.methods[cb_methodname], self.methods[eb_methodname], obj, *args) - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> defer.Deferred: return self._process_parallel('open_spider', spider) - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> defer.Deferred: return self._process_parallel('close_spider', spider) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index c382a00f7..095eae94c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -5,7 +5,7 @@ import asyncio import inspect from collections.abc import Coroutine from functools import wraps -from typing import Callable, Iterable, Any +from typing import Callable, Iterable, Any, Generator from twisted.internet import defer, task from twisted.python import failure @@ -15,7 +15,7 @@ from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import is_asyncio_reactor_installed -def defer_fail(_failure: Failure): +def defer_fail(_failure: Failure) -> defer.Deferred: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -28,7 +28,7 @@ def defer_fail(_failure: Failure): return d -def defer_succeed(result): +def defer_succeed(result) -> defer.Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -41,7 +41,7 @@ def defer_succeed(result): return d -def defer_result(result): +def defer_result(result) -> defer.Deferred: if isinstance(result, defer.Deferred): return result elif isinstance(result, failure.Failure): @@ -50,7 +50,7 @@ def defer_result(result): return defer_succeed(result) -def mustbe_deferred(f: Callable, *args, **kw): +def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -67,7 +67,7 @@ def mustbe_deferred(f: Callable, *args, **kw): return defer_result(result) -def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named): +def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> defer.DeferredList: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -78,7 +78,7 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) -def process_chain(callbacks: Iterable[Callable], input, *a, **kw): +def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: """Return a Deferred built by chaining the given callbacks""" d = defer.Deferred() for x in callbacks: @@ -87,7 +87,7 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw): return d -def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw): +def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" d = defer.Deferred() for cb, eb in zip(callbacks, errbacks): @@ -103,7 +103,7 @@ def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callabl return d -def process_parallel(callbacks: Iterable[Callable], input, *a, **kw): +def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ @@ -113,7 +113,7 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw): return d -def iter_errback(iterable: Iterable, errback: Callable, *a, **kw): +def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -142,7 +142,7 @@ def deferred_from_coro(o) -> Any: return o -def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]): +def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: """ Converts a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. @@ -154,7 +154,7 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]): return f -def maybeDeferred_coro(f: Callable, *args, **kw): +def maybeDeferred_coro(f: Callable, *args, **kw) -> defer.Deferred: """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ try: result = f(*args, **kw) From 335a25675278543d4c85123bbbed99f228b26416 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 13 Apr 2021 21:05:20 +0500 Subject: [PATCH 518/568] Update scrapy/core/spidermw.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/core/spidermw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 05df8c988..7e58521ac 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ from itertools import islice -from typing import Callable, Union, Any, Generator, Iterable +from typing import Any, Callable, Generator, Iterable, Union from twisted.internet.defer import Deferred from twisted.python.failure import Failure From b0e75125749ec1dce14614468bf06cd8e8842649 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 13 Apr 2021 21:05:25 +0500 Subject: [PATCH 519/568] Update scrapy/middleware.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 3f8c1cbf5..09768b59d 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Dict, Deque +from typing import Callable, Deque, Dict from twisted.internet import defer From a8de04c823f5e10d012aaa2dd94a4f5a9f70b119 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 13 Apr 2021 21:05:30 +0500 Subject: [PATCH 520/568] Update scrapy/utils/defer.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/defer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 095eae94c..e1139b1d1 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -5,7 +5,7 @@ import asyncio import inspect from collections.abc import Coroutine from functools import wraps -from typing import Callable, Iterable, Any, Generator +from typing import Any, Callable, Generator, Iterable from twisted.internet import defer, task from twisted.python import failure From cef0a8b3d653d847efe32dfc2850e5992b627408 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Apr 2021 21:07:07 +0500 Subject: [PATCH 521/568] Import Deferred directly in scrapy/middleware.py. --- scrapy/middleware.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 09768b59d..bbec38086 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -3,7 +3,7 @@ import pprint from collections import defaultdict, deque from typing import Callable, Deque, Dict -from twisted.internet import defer +from twisted.internet.defer import Deferred from scrapy import Spider from scrapy.exceptions import NotConfigured @@ -63,18 +63,18 @@ class MiddlewareManager: if hasattr(mw, 'close_spider'): self.methods['close_spider'].appendleft(mw.close_spider) - def _process_parallel(self, methodname: str, obj, *args) -> defer.Deferred: + def _process_parallel(self, methodname: str, obj, *args) -> Deferred: return process_parallel(self.methods[methodname], obj, *args) - def _process_chain(self, methodname: str, obj, *args) -> defer.Deferred: + def _process_chain(self, methodname: str, obj, *args) -> Deferred: return process_chain(self.methods[methodname], obj, *args) - def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> defer.Deferred: + def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: return process_chain_both(self.methods[cb_methodname], self.methods[eb_methodname], obj, *args) - def open_spider(self, spider: Spider) -> defer.Deferred: + def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) - def close_spider(self, spider: Spider) -> defer.Deferred: + def close_spider(self, spider: Spider) -> Deferred: return self._process_parallel('close_spider', spider) From 08e4eaf97369ba6daa4b5d84e00fd4d36b78e00a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Apr 2021 22:41:01 +0500 Subject: [PATCH 522/568] Import Deferred directly in scrapy/utils/defer.py. --- scrapy/utils/defer.py | 48 ++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index e1139b1d1..b317c12a3 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -7,7 +7,9 @@ from collections.abc import Coroutine from functools import wraps from typing import Any, Callable, Generator, Iterable -from twisted.internet import defer, task +from twisted.internet import defer +from twisted.internet.defer import Deferred, DeferredList, ensureDeferred +from twisted.internet.task import Cooperator from twisted.python import failure from twisted.python.failure import Failure @@ -15,7 +17,7 @@ from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import is_asyncio_reactor_installed -def defer_fail(_failure: Failure) -> defer.Deferred: +def defer_fail(_failure: Failure) -> Deferred: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -23,12 +25,12 @@ def defer_fail(_failure: Failure) -> defer.Deferred: before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.errback, _failure) return d -def defer_succeed(result) -> defer.Deferred: +def defer_succeed(result) -> Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -36,13 +38,13 @@ def defer_succeed(result) -> defer.Deferred: before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.callback, result) return d -def defer_result(result) -> defer.Deferred: - if isinstance(result, defer.Deferred): +def defer_result(result) -> Deferred: + if isinstance(result, Deferred): return result elif isinstance(result, failure.Failure): return defer_fail(result) @@ -50,7 +52,7 @@ def defer_result(result) -> defer.Deferred: return defer_succeed(result) -def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred: +def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -67,29 +69,29 @@ def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred: return defer_result(result) -def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> defer.DeferredList: +def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> DeferredList: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. Taken from: https://jcalderone.livejournal.com/24285.html """ - coop = task.Cooperator() + coop = Cooperator() work = (callable(elem, *args, **named) for elem in iterable) - return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) + return DeferredList([coop.coiterate(work) for _ in range(count)]) -def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: +def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks""" - d = defer.Deferred() + d = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) return d -def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: +def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" - d = defer.Deferred() + d = Deferred() for cb, eb in zip(callbacks, errbacks): d.addCallbacks( callback=cb, errback=eb, @@ -103,12 +105,12 @@ def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callabl return d -def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: +def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = defer.DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) + d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d @@ -129,16 +131,16 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: def deferred_from_coro(o) -> Any: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" - if isinstance(o, defer.Deferred): + if isinstance(o, Deferred): return o if asyncio.isfuture(o) or inspect.isawaitable(o): if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" - return defer.ensureDeferred(o) + return ensureDeferred(o) else: # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - return defer.Deferred.fromFuture(asyncio.ensure_future(o)) + return Deferred.fromFuture(asyncio.ensure_future(o)) return o @@ -154,14 +156,14 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: return f -def maybeDeferred_coro(f: Callable, *args, **kw) -> defer.Deferred: +def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ try: result = f(*args, **kw) except: # noqa: E722 - return defer.fail(failure.Failure(captureVars=defer.Deferred.debug)) + return defer.fail(failure.Failure(captureVars=Deferred.debug)) - if isinstance(result, defer.Deferred): + if isinstance(result, Deferred): return result elif asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) From 77bff0db0a6bdfca15295444f0e0eda47b35e702 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Apr 2021 19:10:13 +0500 Subject: [PATCH 523/568] Additional typing for scraper and a small code change. --- scrapy/core/scraper.py | 100 ++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index c760a4155..adbf8ef3d 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -3,11 +3,10 @@ extracts information from them""" import logging from collections import deque -from collections.abc import Iterable -from typing import Union +from typing import Union, Optional, Tuple, Set, Deque, Any, Iterable from itemadapter import is_item -from twisted.internet import defer +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import signals, Spider @@ -20,6 +19,9 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output +QUEUE_TUPLE = Tuple[Union[Response, Failure], Request, Deferred] + + logger = logging.getLogger(__name__) @@ -28,46 +30,46 @@ class Slot: MIN_RESPONSE_SIZE = 1024 - def __init__(self, max_active_size=5000000): + def __init__(self, max_active_size: int = 5000000): self.max_active_size = max_active_size - self.queue = deque() - self.active = set() - self.active_size = 0 - self.itemproc_size = 0 - self.closing = None + self.queue: Deque[QUEUE_TUPLE] = deque() + self.active: Set[Request] = set() + self.active_size: int = 0 + self.itemproc_size: int = 0 + self.closing: Optional[Deferred] = None - def add_response_request(self, response, request): - deferred = defer.Deferred() - self.queue.append((response, request, deferred)) - if isinstance(response, Response): - self.active_size += max(len(response.body), self.MIN_RESPONSE_SIZE) + def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred: + deferred = Deferred() + self.queue.append((result, request, deferred)) + if isinstance(result, Response): + self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) else: self.active_size += self.MIN_RESPONSE_SIZE return deferred - def next_response_request_deferred(self): + def next_response_request_deferred(self) -> QUEUE_TUPLE: response, request, deferred = self.queue.popleft() self.active.add(request) return response, request, deferred - def finish_response(self, response, request): + def finish_response(self, result: Union[Response, Failure], request: Request) -> None: self.active.remove(request) - if isinstance(response, Response): - self.active_size -= max(len(response.body), self.MIN_RESPONSE_SIZE) + if isinstance(result, Response): + self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE) else: self.active_size -= self.MIN_RESPONSE_SIZE - def is_idle(self): + def is_idle(self) -> bool: return not (self.queue or self.active) - def needs_backout(self): + def needs_backout(self) -> bool: return self.active_size > self.max_active_size class Scraper: def __init__(self, crawler): - self.slot = None + self.slot: Optional[Slot] = None self.spidermw = SpiderMiddlewareManager.from_crawler(crawler) itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR']) self.itemproc = itemproc_cls.from_crawler(crawler) @@ -76,32 +78,37 @@ class Scraper: self.signals = crawler.signals self.logformatter = crawler.logformatter - @defer.inlineCallbacks - def open_spider(self, spider): + @inlineCallbacks + def open_spider(self, spider: Spider): """Open the given spider for scraping and allocate resources for it""" self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE')) yield self.itemproc.open_spider(spider) - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> Deferred: """Close a spider being scraped and release its resources""" - self.slot.closing = defer.Deferred() + if self.slot is None: + raise RuntimeError("Scraper slot not assigned") + self.slot.closing = Deferred() self.slot.closing.addCallback(self.itemproc.close_spider) self._check_if_closing(spider) return self.slot.closing - def is_idle(self): + def is_idle(self) -> bool: """Return True if there isn't any more spiders to process""" return not self.slot - def _check_if_closing(self, spider): + def _check_if_closing(self, spider: Spider) -> None: + assert self.slot is not None # typing if self.slot.closing and self.slot.is_idle(): self.slot.closing.callback(spider) - def enqueue_scrape(self, response, request, spider): - dfd = self.slot.add_response_request(response, request) + def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: + if self.slot is None: + raise RuntimeError("Scraper slot not assigned") + dfd = self.slot.add_response_request(result, request) def finish_scraping(_): - self.slot.finish_response(response, request) + self.slot.finish_response(result, request) self._check_if_closing(spider) self._scrape_next(spider) return _ @@ -115,12 +122,13 @@ class Scraper: self._scrape_next(spider) return dfd - def _scrape_next(self, spider): + def _scrape_next(self, spider: Spider) -> None: + assert self.slot is not None # typing while self.slot.queue: response, request, deferred = self.slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider): + def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: """ Handle the downloaded response or failure through the spider callback/errback """ @@ -131,7 +139,7 @@ class Scraper: dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider): + def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: """ Handle the different cases of request's result been a Response or a Failure """ @@ -141,7 +149,7 @@ class Scraper: dfd = self.call_spider(result, request, spider) return dfd.addErrback(self._log_download_errors, result, request, spider) - def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider): + def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request @@ -156,7 +164,7 @@ class Scraper: dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) - def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider): + def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None: exc = _failure.value if isinstance(exc, CloseSpider): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') @@ -177,7 +185,7 @@ class Scraper: spider=spider ) - def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider): + def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred: if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, response, spider) @@ -185,10 +193,12 @@ class Scraper: request, response, spider) return dfd - def _process_spidermw_output(self, output, request, response, spider): + def _process_spidermw_output(self, output: Any, request: Request, response: Response, + spider: Spider) -> Optional[Deferred]: """Process each Request/Item (given in the output parameter) returned from the given spider """ + assert self.slot is not None # typing if isinstance(output, Request): self.crawler.engine.crawl(request=output, spider=spider) elif is_item(output): @@ -205,12 +215,18 @@ class Scraper: {'request': request, 'typename': typename}, extra={'spider': spider}, ) + return None - def _log_download_errors(self, spider_failure, download_failure, request, spider): + def _log_download_errors(self, spider_failure: Failure, download_failure: Failure, request: Request, + spider: Spider) -> Union[Failure, None]: """Log and silence errors that come from the engine (typically download - errors that got propagated thru here) + errors that got propagated thru here). + + spider_failure: the value passed into the errback of self.call_spider() + download_failure: the value passed into _scrape2() from + ExecutionEngine._handle_downloader_output() as "result" """ - if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest): + if not download_failure.check(IgnoreRequest): if download_failure.frames: logkws = self.logformatter.download_error(download_failure, request, spider) logger.log( @@ -230,10 +246,12 @@ class Scraper: if spider_failure is not download_failure: return spider_failure + return None - def _itemproc_finished(self, output, item, response, spider): + def _itemproc_finished(self, output: Any, item: Any, response: Response, spider: Spider) -> None: """ItemProcessor finished for the given ``item`` and returned ``output`` """ + assert self.slot is not None # typing self.slot.itemproc_size -= 1 if isinstance(output, Failure): ex = output.value From 309a637f32b0f6196eb4f77f19152527dcf3e5e7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Apr 2021 20:26:37 +0500 Subject: [PATCH 524/568] Small changes. --- scrapy/core/scraper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index adbf8ef3d..96aa53686 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -3,7 +3,7 @@ extracts information from them""" import logging from collections import deque -from typing import Union, Optional, Tuple, Set, Deque, Any, Iterable +from typing import Any, Deque, Iterable, Optional, Set, Tuple, Union from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks @@ -19,7 +19,7 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output -QUEUE_TUPLE = Tuple[Union[Response, Failure], Request, Deferred] +QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] logger = logging.getLogger(__name__) @@ -32,7 +32,7 @@ class Slot: def __init__(self, max_active_size: int = 5000000): self.max_active_size = max_active_size - self.queue: Deque[QUEUE_TUPLE] = deque() + self.queue: Deque[QueueTuple] = deque() self.active: Set[Request] = set() self.active_size: int = 0 self.itemproc_size: int = 0 @@ -47,7 +47,7 @@ class Slot: self.active_size += self.MIN_RESPONSE_SIZE return deferred - def next_response_request_deferred(self) -> QUEUE_TUPLE: + def next_response_request_deferred(self) -> QueueTuple: response, request, deferred = self.queue.popleft() self.active.add(request) return response, request, deferred From 7e23677b52b659b11471a63f3be9905a0bbaf995 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 20 Apr 2021 08:45:28 -0300 Subject: [PATCH 525/568] Engine: deprecations and type hints (#5090) --- docs/topics/telnetconsole.rst | 3 +- scrapy/core/engine.py | 419 ++++++++++--------- scrapy/core/scraper.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 2 +- scrapy/extensions/memusage.py | 6 +- scrapy/pipelines/media.py | 2 +- scrapy/shell.py | 2 +- scrapy/utils/engine.py | 3 +- tests/test_downloadermiddleware_robotstxt.py | 12 +- tests/test_engine.py | 110 ++++- 10 files changed, 336 insertions(+), 225 deletions(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 9802a34a2..832829b75 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -110,11 +110,10 @@ using the telnet console:: Execution engine status time()-engine.start_time : 8.62972998619 - engine.has_capacity() : False len(engine.downloader.active) : 16 engine.scraper.is_idle() : False engine.spider.name : followall - engine.spider_is_idle(engine.spider) : False + engine.spider_is_idle() : False engine.slot.closing : False len(engine.slot.inprogress) : 16 len(engine.slot.scheduler.dqs or []) : 0 diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 93bcdb49a..edfac87c6 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -1,51 +1,61 @@ """ -This is the Scrapy engine which controls the Scheduler, Downloader and Spiders. +This is the Scrapy engine which controls the Scheduler, Downloader and Spider. For more information see docs/topics/architecture.rst """ import logging +import warnings from time import time +from typing import Callable, Iterable, Iterator, Optional, Set, Union -from twisted.internet import defer, task +from twisted.internet.defer import Deferred, inlineCallbacks, succeed +from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper -from scrapy.exceptions import DontCloseSpider +from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning from scrapy.http import Response, Request -from scrapy.utils.misc import load_object -from scrapy.utils.reactor import CallLaterOnce +from scrapy.spiders import Spider from scrapy.utils.log import logformatter_adapter, failure_to_exc_info +from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.reactor import CallLaterOnce + logger = logging.getLogger(__name__) class Slot: - - def __init__(self, start_requests, close_if_idle, nextcall, scheduler): - self.closing = False - self.inprogress = set() # requests in progress - self.start_requests = iter(start_requests) + def __init__( + self, + start_requests: Iterable, + close_if_idle: bool, + nextcall: CallLaterOnce, + scheduler, + ) -> None: + self.closing: Optional[Deferred] = None + self.inprogress: Set[Request] = set() + self.start_requests: Optional[Iterator] = iter(start_requests) self.close_if_idle = close_if_idle self.nextcall = nextcall self.scheduler = scheduler - self.heartbeat = task.LoopingCall(nextcall.schedule) + self.heartbeat = LoopingCall(nextcall.schedule) - def add_request(self, request): + def add_request(self, request: Request) -> None: self.inprogress.add(request) - def remove_request(self, request): + def remove_request(self, request: Request) -> None: self.inprogress.remove(request) self._maybe_fire_closing() - def close(self): - self.closing = defer.Deferred() + def close(self) -> Deferred: + self.closing = Deferred() self._maybe_fire_closing() return self.closing - def _maybe_fire_closing(self): - if self.closing and not self.inprogress: + def _maybe_fire_closing(self) -> None: + if self.closing is not None and not self.inprogress: if self.nextcall: self.nextcall.cancel() if self.heartbeat.running: @@ -54,210 +64,224 @@ class Slot: class ExecutionEngine: - - def __init__(self, crawler, spider_closed_callback): + def __init__(self, crawler, spider_closed_callback: Callable) -> None: self.crawler = crawler self.settings = crawler.settings self.signals = crawler.signals self.logformatter = crawler.logformatter - self.slot = None - self.spider = None + self.slot: Optional[Slot] = None + self.spider: Optional[Spider] = None self.running = False self.paused = False - self.scheduler_cls = load_object(self.settings['SCHEDULER']) + self.scheduler_cls = load_object(crawler.settings["SCHEDULER"]) downloader_cls = load_object(self.settings['DOWNLOADER']) self.downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback = spider_closed_callback - @defer.inlineCallbacks - def start(self): - """Start the execution engine""" + @inlineCallbacks + def start(self) -> Deferred: if self.running: raise RuntimeError("Engine already running") self.start_time = time() yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True - self._closewait = defer.Deferred() + self._closewait = Deferred() yield self._closewait - def stop(self): - """Stop the execution engine gracefully""" + def stop(self) -> Deferred: + """Gracefully stop the execution engine""" + @inlineCallbacks + def _finish_stopping_engine(_) -> Deferred: + yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + self._closewait.callback(None) + if not self.running: raise RuntimeError("Engine not running") + self.running = False - dfd = self._close_all_spiders() - return dfd.addBoth(lambda _: self._finish_stopping_engine()) + dfd = self.close_spider(self.spider, reason="shutdown") if self.spider is not None else succeed(None) + return dfd.addBoth(_finish_stopping_engine) - def close(self): - """Close the execution engine gracefully. - - If it has already been started, stop it. In all cases, close all spiders - and the downloader. + def close(self) -> Deferred: + """ + Gracefully close the execution engine. + If it has already been started, stop it. In all cases, close the spider and the downloader. """ if self.running: - # Will also close spiders and downloader - return self.stop() - elif self.open_spiders: - # Will also close downloader - return self._close_all_spiders() - else: - return defer.succeed(self.downloader.close()) + return self.stop() # will also close spider and downloader + if self.spider is not None: + return self.close_spider(self.spider, reason="shutdown") # will also close downloader + return succeed(self.downloader.close()) - def pause(self): - """Pause the execution engine""" + def pause(self) -> None: self.paused = True - def unpause(self): - """Resume the execution engine""" + def unpause(self) -> None: self.paused = False - def _next_request(self, spider): - slot = self.slot - if not slot: - return + def _next_request(self) -> None: + assert self.slot is not None # typing + assert self.spider is not None # typing if self.paused: - return + return None - while not self._needs_backout(spider): - if not self._next_request_from_scheduler(spider): - break + while not self._needs_backout() and self._next_request_from_scheduler() is not None: + pass - if slot.start_requests and not self._needs_backout(spider): + if self.slot.start_requests is not None and not self._needs_backout(): try: - request = next(slot.start_requests) + request = next(self.slot.start_requests) except StopIteration: - slot.start_requests = None + self.slot.start_requests = None except Exception: - slot.start_requests = None - logger.error('Error while obtaining start requests', - exc_info=True, extra={'spider': spider}) + self.slot.start_requests = None + logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': self.spider}) else: - self.crawl(request, spider) + self.crawl(request) - if self.spider_is_idle(spider) and slot.close_if_idle: - self._spider_idle(spider) + if self.spider_is_idle() and self.slot.close_if_idle: + self._spider_idle() - def _needs_backout(self, spider): - slot = self.slot + def _needs_backout(self) -> bool: return ( not self.running - or slot.closing + or self.slot.closing # type: ignore[union-attr] or self.downloader.needs_backout() - or self.scraper.slot.needs_backout() + or self.scraper.slot.needs_backout() # type: ignore[union-attr] ) - def _next_request_from_scheduler(self, spider): - slot = self.slot - request = slot.scheduler.next_request() - if not request: - return - d = self._download(request, spider) - d.addBoth(self._handle_downloader_output, request, spider) + def _next_request_from_scheduler(self) -> Optional[Deferred]: + assert self.slot is not None # typing + assert self.spider is not None # typing + + request = self.slot.scheduler.next_request() + if request is None: + return None + + d = self._download(request, self.spider) + d.addBoth(self._handle_downloader_output, request, self.spider) d.addErrback(lambda f: logger.info('Error while handling downloader output', exc_info=failure_to_exc_info(f), - extra={'spider': spider})) - d.addBoth(lambda _: slot.remove_request(request)) + extra={'spider': self.spider})) + d.addBoth(lambda _: self.slot.remove_request(request)) d.addErrback(lambda f: logger.info('Error while removing request from slot', exc_info=failure_to_exc_info(f), - extra={'spider': spider})) - d.addBoth(lambda _: slot.nextcall.schedule()) + extra={'spider': self.spider})) + d.addBoth(lambda _: self.slot.nextcall.schedule()) d.addErrback(lambda f: logger.info('Error while scheduling new request', exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + extra={'spider': self.spider})) return d - def _handle_downloader_output(self, response, request, spider): - if not isinstance(response, (Request, Response, Failure)): - raise TypeError( - "Incorrect type: expected Request, Response or Failure, got " - f"{type(response)}: {response!r}" - ) + def _handle_downloader_output( + self, result: Union[Request, Response, Failure], request: Request, spider: Spider + ) -> Optional[Deferred]: + if not isinstance(result, (Request, Response, Failure)): + raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}") + # downloader middleware can return requests (for example, redirects) - if isinstance(response, Request): - self.crawl(response, spider) - return - # response is a Response or Failure - d = self.scraper.enqueue_scrape(response, request, spider) - d.addErrback(lambda f: logger.error('Error while enqueuing downloader output', - exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + if isinstance(result, Request): + self.crawl(result) + return None + + d = self.scraper.enqueue_scrape(result, request, spider) + d.addErrback( + lambda f: logger.error( + "Error while enqueuing downloader output", + exc_info=failure_to_exc_info(f), + extra={'spider': spider}, + ) + ) return d - def spider_is_idle(self, spider): - if not self.scraper.slot.is_idle(): - # scraper is not idle + def spider_is_idle(self, spider: Optional[Spider] = None) -> bool: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: + raise RuntimeError("Engine slot not assigned") + if not self.scraper.slot.is_idle(): # type: ignore[union-attr] return False - - if self.downloader.active: - # downloader has pending requests + if self.downloader.active: # downloader has pending requests return False - - if self.slot.start_requests is not None: - # not all start requests are handled + if self.slot.start_requests is not None: # not all start requests are handled return False - if self.slot.scheduler.has_pending_requests(): - # scheduler has pending requests return False - return True - @property - def open_spiders(self): - return [self.spider] if self.spider else [] + def crawl(self, request: Request, spider: Optional[Spider] = None) -> None: + """Inject the request into the spider <-> downloader pipeline""" + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if spider is not self.spider: + raise RuntimeError(f"The spider {spider.name!r} does not match the open spider") + if self.spider is None: + raise RuntimeError(f"No open spider to crawl: {request}") + self._schedule_request(request, self.spider) + self.slot.nextcall.schedule() # type: ignore[union-attr] - def has_capacity(self): - """Does the engine have capacity to handle more spiders""" - return not bool(self.slot) - - def crawl(self, request, spider): - if spider not in self.open_spiders: - raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}") - self.schedule(request, spider) - self.slot.nextcall.schedule() - - def schedule(self, request, spider): + def _schedule_request(self, request: Request, spider: Spider) -> None: self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider) - if not self.slot.scheduler.enqueue_request(request): + if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider) - def download(self, request, spider): - d = self._download(request, spider) - d.addBoth(self._downloaded, self.slot, request, spider) - return d + def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred: + """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" + if spider is None: + spider = self.spider + else: + warnings.warn( + "Passing a 'spider' argument to ExecutionEngine.download is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if spider is not self.spider: + logger.warning("The spider '%s' does not match the open spider", spider.name) + if spider is None: + raise RuntimeError(f"No open spider to crawl: {request}") + return self._download(request, spider).addBoth(self._downloaded, request, spider) - def _downloaded(self, response, slot, request, spider): - slot.remove_request(request) - return self.download(response, spider) if isinstance(response, Request) else response + def _downloaded( + self, result: Union[Response, Request], request: Request, spider: Spider + ) -> Union[Deferred, Response]: + assert self.slot is not None # typing + self.slot.remove_request(request) + return self.download(result, spider) if isinstance(result, Request) else result - def _download(self, request, spider): - slot = self.slot - slot.add_request(request) + def _download(self, request: Request, spider: Spider) -> Deferred: + assert self.slot is not None # typing - def _on_success(response): - if not isinstance(response, (Response, Request)): - raise TypeError( - "Incorrect type: expected Response or Request, got " - f"{type(response)}: {response!r}" - ) - if isinstance(response, Response): - if response.request is None: - response.request = request - logkws = self.logformatter.crawled(response.request, response, spider) + self.slot.add_request(request) + + def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: + if not isinstance(result, (Response, Request)): + raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}") + if isinstance(result, Response): + if result.request is None: + result.request = request + logkws = self.logformatter.crawled(result.request, result, spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) self.signals.send_catch_log( signal=signals.response_received, - response=response, - request=response.request, + response=result, + request=result.request, spider=spider, ) - return response + return result def _on_complete(_): - slot.nextcall.schedule() + self.slot.nextcall.schedule() return _ dwld = self.downloader.fetch(request, spider) @@ -265,58 +289,52 @@ class ExecutionEngine: dwld.addBoth(_on_complete) return dwld - @defer.inlineCallbacks - def open_spider(self, spider, start_requests=(), close_if_idle=True): - if not self.has_capacity(): + @inlineCallbacks + def open_spider(self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True): + if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={'spider': spider}) - nextcall = CallLaterOnce(self._next_request, spider) - scheduler = self.scheduler_cls.from_crawler(self.crawler) + nextcall = CallLaterOnce(self._next_request) + scheduler = create_instance(self.scheduler_cls, settings=None, crawler=self.crawler) start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider) - slot = Slot(start_requests, close_if_idle, nextcall, scheduler) - self.slot = slot + self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider yield scheduler.open(spider) yield self.scraper.open_spider(spider) self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) - slot.nextcall.schedule() - slot.heartbeat.start(5) + self.slot.nextcall.schedule() + self.slot.heartbeat.start(5) - def _spider_idle(self, spider): - """Called when a spider gets idle. This function is called when there - are no remaining pages to download or schedule. It can be called - multiple times. If some extension raises a DontCloseSpider exception - (in the spider_idle signal handler) the spider is not closed until the - next loop and this function is guaranteed to be called (at least) once - again for this spider. + def _spider_idle(self) -> None: """ - res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider) + Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. + It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider + exception, the spider is not closed until the next loop and this function is guaranteed to be called + (at least) once again. + """ + assert self.spider is not None # typing + res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider) if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res): - return + return None + if self.spider_is_idle(): + self.close_spider(self.spider, reason='finished') - if self.spider_is_idle(spider): - self.close_spider(spider, reason='finished') - - def close_spider(self, spider, reason='cancelled'): + def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: """Close (cancel) spider and clear all its outstanding requests""" + if self.slot is None: + raise RuntimeError("Engine slot not assigned") - slot = self.slot - if slot.closing: - return slot.closing - logger.info("Closing spider (%(reason)s)", - {'reason': reason}, - extra={'spider': spider}) + if self.slot.closing is not None: + return self.slot.closing - dfd = slot.close() + logger.info("Closing spider (%(reason)s)", {'reason': reason}, extra={'spider': spider}) - def log_failure(msg): - def errback(failure): - logger.error( - msg, - exc_info=failure_to_exc_info(failure), - extra={'spider': spider} - ) + dfd = self.slot.close() + + def log_failure(msg: str) -> Callable: + def errback(failure: Failure) -> None: + logger.error(msg, exc_info=failure_to_exc_info(failure), extra={'spider': spider}) return errback dfd.addBoth(lambda _: self.downloader.close()) @@ -325,19 +343,18 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.scraper.close_spider(spider)) dfd.addErrback(log_failure('Scraper close failure')) - dfd.addBoth(lambda _: slot.scheduler.close(reason)) + dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) dfd.addErrback(log_failure('Scheduler close failure')) dfd.addBoth(lambda _: self.signals.send_catch_log_deferred( - signal=signals.spider_closed, spider=spider, reason=reason)) + signal=signals.spider_closed, spider=spider, reason=reason, + )) dfd.addErrback(log_failure('Error while sending spider_close signal')) dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason)) dfd.addErrback(log_failure('Stats close failure')) - dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", - {'reason': reason}, - extra={'spider': spider})) + dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", {'reason': reason}, extra={'spider': spider})) dfd.addBoth(lambda _: setattr(self, 'slot', None)) dfd.addErrback(log_failure('Error while unassigning slot')) @@ -349,12 +366,26 @@ class ExecutionEngine: return dfd - def _close_all_spiders(self): - dfds = [self.close_spider(s, reason='shutdown') for s in self.open_spiders] - dlist = defer.DeferredList(dfds) - return dlist + @property + def open_spiders(self) -> list: + warnings.warn( + "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return [self.spider] if self.spider is not None else [] - @defer.inlineCallbacks - def _finish_stopping_engine(self): - yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) - self._closewait.callback(None) + def has_capacity(self) -> bool: + warnings.warn("ExecutionEngine.has_capacity is deprecated", ScrapyDeprecationWarning, stacklevel=2) + return not bool(self.slot) + + def schedule(self, request: Request, spider: Spider) -> None: + warnings.warn( + "ExecutionEngine.schedule is deprecated, please use " + "ExecutionEngine.crawl or ExecutionEngine.download instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: + raise RuntimeError("Engine slot not assigned") + self._schedule_request(request, spider) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 96aa53686..d6d6f64f9 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -200,7 +200,7 @@ class Scraper: """ assert self.slot is not None # typing if isinstance(output, Request): - self.crawler.engine.crawl(request=output, spider=spider) + self.crawler.engine.crawl(request=output) elif is_item(output): self.slot.itemproc_size += 1 dfd = self.itemproc.process_item(output, spider) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index d6da55535..e66bf177e 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -67,7 +67,7 @@ class RobotsTxtMiddleware: priority=self.DOWNLOAD_PRIORITY, meta={'dont_obey_robotstxt': True} ) - dfd = self.crawler.engine.download(robotsreq, spider) + dfd = self.crawler.engine.download(robotsreq) dfd.addCallback(self._parse_robots, netloc, spider) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 274cbdbfe..9de119a10 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -88,10 +88,8 @@ class MemoryUsage: self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) - open_spiders = self.crawler.engine.open_spiders - if open_spiders: - for spider in open_spiders: - self.crawler.engine.close_spider(spider, 'memusage_exceeded') + if self.crawler.engine.spider is not None: + self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded') else: self.crawler.stop() diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0c2ee6856..d1bccf323 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -173,7 +173,7 @@ class MediaPipeline: errback=self.media_failed, errbackArgs=(request, info)) else: self._modify_media_request(request) - dfd = self.crawler.engine.download(request, info.spider) + dfd = self.crawler.engine.download(request) dfd.addCallbacks( callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) diff --git a/scrapy/shell.py b/scrapy/shell.py index c370ccaff..f2dff2ae3 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -79,7 +79,7 @@ class Shell: spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) - self.crawler.engine.crawl(request, spider) + self.crawler.engine.crawl(request) return d def _open_spider(self, request, spider): diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 0c1cee1a0..8e3ec2c37 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -8,11 +8,10 @@ def get_engine_status(engine): """Return a report of the current engine status""" tests = [ "time()-engine.start_time", - "engine.has_capacity()", "len(engine.downloader.active)", "engine.scraper.is_idle()", "engine.spider.name", - "engine.spider_is_idle(engine.spider)", + "engine.spider_is_idle()", "engine.slot.closing", "len(engine.slot.inprogress)", "len(engine.slot.scheduler.dqs or [])", diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 858138f81..1460d88eb 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -42,7 +42,7 @@ Disallow: /some/randome/page.html """.encode('utf-8') response = TextResponse('http://site.local/robots.txt', body=ROBOTS) - def return_response(request, spider): + def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -79,7 +79,7 @@ Disallow: /some/randome/page.html crawler.settings.set('ROBOTSTXT_OBEY', True) response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2') - def return_response(request, spider): + def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -102,7 +102,7 @@ Disallow: /some/randome/page.html crawler.settings.set('ROBOTSTXT_OBEY', True) response = Response('http://site.local/robots.txt') - def return_response(request, spider): + def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -122,7 +122,7 @@ Disallow: /some/randome/page.html self.crawler.settings.set('ROBOTSTXT_OBEY', True) err = error.DNSLookupError('Robotstxt address not found') - def return_failure(request, spider): + def return_failure(request): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(err)) return deferred @@ -138,7 +138,7 @@ Disallow: /some/randome/page.html self.crawler.settings.set('ROBOTSTXT_OBEY', True) err = error.DNSLookupError('Robotstxt address not found') - def immediate_failure(request, spider): + def immediate_failure(request): deferred = Deferred() deferred.errback(failure.Failure(err)) return deferred @@ -150,7 +150,7 @@ Disallow: /some/randome/page.html def test_ignore_robotstxt_request(self): self.crawler.settings.set('ROBOTSTXT_OBEY', True) - def ignore_request(request, spider): + def ignore_request(request): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest())) return deferred diff --git a/tests/test_engine.py b/tests/test_engine.py index b2d1d83c7..c200ded90 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -13,6 +13,7 @@ module with the ``runserver`` argument:: import os import re import sys +import warnings from collections import defaultdict from urllib.parse import urlparse @@ -25,6 +26,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor @@ -382,22 +384,104 @@ class EngineTest(unittest.TestCase): yield e.close() @defer.inlineCallbacks - def test_close_spiders_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - self.assertEqual(len(e.open_spiders), 1) - yield e.close() - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_close_engine_spiders_downloader(self): + def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() - self.assertTrue(e.running) - yield e.close() - self.assertFalse(e.running) - self.assertEqual(len(e.open_spiders), 0) + yield self.assertFailure(e.start(), RuntimeError).addBoth( + lambda exc: self.assertEqual(str(exc), "Engine already running") + ) + yield e.stop() + + @defer.inlineCallbacks + def test_close_spiders_downloader(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + yield e.open_spider(TestSpider(), []) + self.assertEqual(len(e.open_spiders), 1) + yield e.close() + self.assertEqual(len(e.open_spiders), 0) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", + ) + + @defer.inlineCallbacks + def test_close_engine_spiders_downloader(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + yield e.open_spider(TestSpider(), []) + e.start() + self.assertTrue(e.running) + yield e.close() + self.assertFalse(e.running) + self.assertEqual(len(e.open_spiders), 0) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", + ) + + @defer.inlineCallbacks + def test_crawl_deprecated_spider_arg(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + spider = TestSpider() + yield e.open_spider(spider, []) + e.start() + e.crawl(Request("data:,"), spider) + yield e.close() + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", + ) + + @defer.inlineCallbacks + def test_download_deprecated_spider_arg(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + spider = TestSpider() + yield e.open_spider(spider, []) + e.start() + e.download(Request("data:,"), spider) + yield e.close() + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "Passing a 'spider' argument to ExecutionEngine.download is deprecated", + ) + + @defer.inlineCallbacks + def test_deprecated_schedule(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + spider = TestSpider() + yield e.open_spider(spider, []) + e.start() + e.schedule(Request("data:,"), spider) + yield e.close() + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "ExecutionEngine.schedule is deprecated, please use " + "ExecutionEngine.crawl or ExecutionEngine.download instead", + ) + + @defer.inlineCallbacks + def test_deprecated_has_capacity(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + self.assertTrue(e.has_capacity()) + spider = TestSpider() + yield e.open_spider(spider, []) + self.assertFalse(e.has_capacity()) + e.start() + yield e.close() + self.assertTrue(e.has_capacity()) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated") if __name__ == "__main__": From e3f81d8d5f17515b6eba135ac0db7e270ff0a9f0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 20 Apr 2021 11:46:43 -0300 Subject: [PATCH 526/568] Engine: remove unnecessary parameter (#5106) --- scrapy/core/engine.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index edfac87c6..7a09bafa1 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -161,7 +161,7 @@ class ExecutionEngine: return None d = self._download(request, self.spider) - d.addBoth(self._handle_downloader_output, request, self.spider) + d.addBoth(self._handle_downloader_output, request) d.addErrback(lambda f: logger.info('Error while handling downloader output', exc_info=failure_to_exc_info(f), extra={'spider': self.spider})) @@ -176,8 +176,10 @@ class ExecutionEngine: return d def _handle_downloader_output( - self, result: Union[Request, Response, Failure], request: Request, spider: Spider + self, result: Union[Request, Response, Failure], request: Request ) -> Optional[Deferred]: + assert self.spider is not None # typing + if not isinstance(result, (Request, Response, Failure)): raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}") @@ -186,12 +188,12 @@ class ExecutionEngine: self.crawl(result) return None - d = self.scraper.enqueue_scrape(result, request, spider) + d = self.scraper.enqueue_scrape(result, request, self.spider) d.addErrback( lambda f: logger.error( "Error while enqueuing downloader output", exc_info=failure_to_exc_info(f), - extra={'spider': spider}, + extra={'spider': self.spider}, ) ) return d From e779ed7d93beec36f565d33a1cc8d3e8fe6068d7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 20 Apr 2021 16:39:07 -0300 Subject: [PATCH 527/568] Dupefilter type hints (#5108) --- scrapy/dupefilters.py | 41 +++++++++++++++++++++++++++-------------- scrapy/utils/request.py | 2 +- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index ac5478e7c..292c68099 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,35 +1,47 @@ -import os import logging +import os +from typing import Optional, Set, Type, TypeVar +from twisted.internet.defer import Deferred + +from scrapy.http.request import Request +from scrapy.settings import BaseSettings +from scrapy.spiders import Spider from scrapy.utils.job import job_dir from scrapy.utils.request import referer_str, request_fingerprint -class BaseDupeFilter: +BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter") + +class BaseDupeFilter: @classmethod - def from_settings(cls, settings): + def from_settings(cls: Type[BaseDupeFilterTV], settings: BaseSettings) -> BaseDupeFilterTV: return cls() - def request_seen(self, request): + def request_seen(self, request: Request) -> bool: return False - def open(self): # can return deferred + def open(self) -> Optional[Deferred]: pass - def close(self, reason): # can return a deferred + def close(self, reason: str) -> Optional[Deferred]: pass - def log(self, request, spider): # log that a request has been filtered + def log(self, request: Request, spider: Spider) -> None: + """Log that a request has been filtered""" pass +RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter") + + class RFPDupeFilter(BaseDupeFilter): """Request Fingerprint duplicates filter""" - def __init__(self, path=None, debug=False): + def __init__(self, path: Optional[str] = None, debug: bool = False) -> None: self.file = None - self.fingerprints = set() + self.fingerprints: Set[str] = set() self.logdupes = True self.debug = debug self.logger = logging.getLogger(__name__) @@ -39,26 +51,27 @@ class RFPDupeFilter(BaseDupeFilter): self.fingerprints.update(x.rstrip() for x in self.file) @classmethod - def from_settings(cls, settings): + def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV: debug = settings.getbool('DUPEFILTER_DEBUG') return cls(job_dir(settings), debug) - def request_seen(self, request): + def request_seen(self, request: Request) -> bool: fp = self.request_fingerprint(request) if fp in self.fingerprints: return True self.fingerprints.add(fp) if self.file: self.file.write(fp + '\n') + return False - def request_fingerprint(self, request): + def request_fingerprint(self, request: Request) -> str: return request_fingerprint(request) - def close(self, reason): + def close(self, reason: str) -> None: if self.file: self.file.close() - def log(self, request, spider): + def log(self, request: Request, spider: Spider) -> None: if self.debug: msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)" args = {'request': request, 'referer': referer_str(request)} diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 66736b42f..541368423 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -24,7 +24,7 @@ def request_fingerprint( request: Request, include_headers: Optional[Iterable[Union[bytes, str]]] = None, keep_fragments: bool = False, -): +) -> str: """ Return the request fingerprint. From 68379197986ae3deb81a545b5fd6920ea3347094 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 26 Apr 2021 14:55:02 -0300 Subject: [PATCH 528/568] Add peek method to queues (#5112) --- pylintrc | 1 + scrapy/pqueues.py | 59 +++++++--- scrapy/squeues.py | 59 +++++++--- tests/test_pqueues.py | 144 +++++++++++++++++++++++ tests/test_squeues_request.py | 214 ++++++++++++++++++++++++++++++++++ 5 files changed, 447 insertions(+), 30 deletions(-) create mode 100644 tests/test_pqueues.py create mode 100644 tests/test_squeues_request.py diff --git a/pylintrc b/pylintrc index 5b6b9fab0..972bf99de 100644 --- a/pylintrc +++ b/pylintrc @@ -24,6 +24,7 @@ disable=abstract-method, consider-using-in, consider-using-set-comprehension, consider-using-sys-exit, + consider-using-with, cyclic-import, dangerous-default-value, deprecated-method, diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index a9aa6c649..b4b63e7c7 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -3,6 +3,7 @@ import logging from scrapy.utils.misc import create_instance + logger = logging.getLogger(__name__) @@ -17,8 +18,7 @@ def _path_safe(text): >>> _path_safe('some@symbol?').startswith('some_symbol_') True """ - pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' - for c in text]) + pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part unique_slot = hashlib.md5(text.encode('utf8')).hexdigest() @@ -35,6 +35,9 @@ class ScrapyPriorityQueue: * close() * __len__() + Optionally, the queue could provide a ``peek`` method, that should return the + next object to be returned by ``pop``, but without removing it from the queue. + ``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls argument, which is a class used to instantiate a new (internal) queue when a new priority is allocated. @@ -70,10 +73,12 @@ class ScrapyPriorityQueue: self.curprio = min(startprios) def qfactory(self, key): - return create_instance(self.downstream_queue_cls, - None, - self.crawler, - self.key + '/' + str(key)) + return create_instance( + self.downstream_queue_cls, + None, + self.crawler, + self.key + '/' + str(key), + ) def priority(self, request): return -request.priority @@ -99,6 +104,18 @@ class ScrapyPriorityQueue: self.curprio = min(prios) if prios else None return m + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + if self.curprio is None: + return None + queue = self.queues[self.curprio] + return queue.peek() + def close(self): active = [] for p, q in self.queues.items(): @@ -116,8 +133,7 @@ class DownloaderInterface: self.downloader = crawler.engine.downloader def stats(self, possible_slots): - return [(self._active_downloads(slot), slot) - for slot in possible_slots] + return [(self._active_downloads(slot), slot) for slot in possible_slots] def get_slot_key(self, request): return self.downloader._get_slot_key(request, None) @@ -162,10 +178,12 @@ class DownloaderAwarePriorityQueue: self.pqueues[slot] = self.pqfactory(slot, startprios) def pqfactory(self, slot, startprios=()): - return ScrapyPriorityQueue(self.crawler, - self.downstream_queue_cls, - self.key + '/' + _path_safe(slot), - startprios) + return ScrapyPriorityQueue( + self.crawler, + self.downstream_queue_cls, + self.key + '/' + _path_safe(slot), + startprios, + ) def pop(self): stats = self._downloader_interface.stats(self.pqueues) @@ -187,9 +205,22 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] queue.push(request) + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + stats = self._downloader_interface.stats(self.pqueues) + if not stats: + return None + slot = min(stats)[1] + queue = self.pqueues[slot] + return queue.peek() + def close(self): - active = {slot: queue.close() - for slot, queue in self.pqueues.items()} + active = {slot: queue.close() for slot, queue in self.pqueues.items()} self.pqueues.clear() return active diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 77ffda6f7..44898ba08 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -19,7 +19,6 @@ def _with_mkdir(queue_class): dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname, exist_ok=True) - super().__init__(path, *args, **kwargs) return DirectoriesCreated @@ -38,6 +37,20 @@ def _serializable_queue(queue_class, serialize, deserialize): if s: return deserialize(s) + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + try: + s = super().peek() + except AttributeError as ex: + raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex + if s: + return deserialize(s) + return SerializableQueue @@ -59,12 +72,21 @@ def _scrapy_serialization_queue(queue_class): def pop(self): request = super().pop() - if not request: return None + return request_from_dict(request, self.spider) - request = request_from_dict(request, self.spider) - return request + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + request = super().peek() + if not request: + return None + return request_from_dict(request, self.spider) return ScrapyRequestQueue @@ -76,6 +98,19 @@ def _scrapy_non_serialization_queue(queue_class): def from_crawler(cls, crawler, *args, **kwargs): return cls() + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + try: + s = super().peek() + except AttributeError as ex: + raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex + return s + return ScrapyRequestQueue @@ -109,17 +144,9 @@ MarshalLifoDiskQueueNonRequest = _serializable_queue( marshal.loads ) -PickleFifoDiskQueue = _scrapy_serialization_queue( - PickleFifoDiskQueueNonRequest -) -PickleLifoDiskQueue = _scrapy_serialization_queue( - PickleLifoDiskQueueNonRequest -) -MarshalFifoDiskQueue = _scrapy_serialization_queue( - MarshalFifoDiskQueueNonRequest -) -MarshalLifoDiskQueue = _scrapy_serialization_queue( - MarshalLifoDiskQueueNonRequest -) +PickleFifoDiskQueue = _scrapy_serialization_queue(PickleFifoDiskQueueNonRequest) +PickleLifoDiskQueue = _scrapy_serialization_queue(PickleLifoDiskQueueNonRequest) +MarshalFifoDiskQueue = _scrapy_serialization_queue(MarshalFifoDiskQueueNonRequest) +MarshalLifoDiskQueue = _scrapy_serialization_queue(MarshalLifoDiskQueueNonRequest) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py new file mode 100644 index 000000000..ec55033d1 --- /dev/null +++ b/tests/test_pqueues.py @@ -0,0 +1,144 @@ +import tempfile +import unittest + +import queuelib + +from scrapy.http.request import Request +from scrapy.pqueues import ScrapyPriorityQueue, DownloaderAwarePriorityQueue +from scrapy.spiders import Spider +from scrapy.squeues import FifoMemoryQueue +from scrapy.utils.test import get_crawler + +from tests.test_scheduler import MockDownloader, MockEngine + + +class PriorityQueueTest(unittest.TestCase): + def setUp(self): + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider("foo") + + def test_queue_push_pop_one(self): + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + self.assertIsNone(queue.pop()) + self.assertEqual(len(queue), 0) + req1 = Request("https://example.org/1", priority=1) + queue.push(req1) + self.assertEqual(len(queue), 1) + dequeued = queue.pop() + self.assertEqual(len(queue), 0) + self.assertEqual(dequeued.url, req1.url) + self.assertEqual(dequeued.priority, req1.priority) + self.assertEqual(queue.close(), []) + + def test_no_peek_raises(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + queue.push(Request("https://example.org")) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + queue.peek() + queue.close() + + def test_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + self.assertEqual(len(queue), 0) + self.assertIsNone(queue.peek()) + req1 = Request("https://example.org/1") + req2 = Request("https://example.org/2") + req3 = Request("https://example.org/3") + queue.push(req1) + queue.push(req2) + queue.push(req3) + self.assertEqual(len(queue), 3) + self.assertEqual(queue.peek().url, req1.url) + self.assertEqual(queue.pop().url, req1.url) + self.assertEqual(len(queue), 2) + self.assertEqual(queue.peek().url, req2.url) + self.assertEqual(queue.pop().url, req2.url) + self.assertEqual(len(queue), 1) + self.assertEqual(queue.peek().url, req3.url) + self.assertEqual(queue.pop().url, req3.url) + self.assertEqual(queue.close(), []) + + def test_queue_push_pop_priorities(self): + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]) + self.assertIsNone(queue.pop()) + self.assertEqual(len(queue), 0) + req1 = Request("https://example.org/1", priority=1) + req2 = Request("https://example.org/2", priority=2) + req3 = Request("https://example.org/3", priority=3) + queue.push(req1) + queue.push(req2) + queue.push(req3) + self.assertEqual(len(queue), 3) + dequeued = queue.pop() + self.assertEqual(len(queue), 2) + self.assertEqual(dequeued.url, req3.url) + self.assertEqual(dequeued.priority, req3.priority) + self.assertEqual(queue.close(), [-1, -2]) + + +class DownloaderAwarePriorityQueueTest(unittest.TestCase): + def setUp(self): + crawler = get_crawler(Spider) + crawler.engine = MockEngine(downloader=MockDownloader()) + self.queue = DownloaderAwarePriorityQueue.from_crawler( + crawler=crawler, + downstream_queue_cls=FifoMemoryQueue, + key="foo/bar", + ) + + def tearDown(self): + self.queue.close() + + def test_push_pop(self): + self.assertEqual(len(self.queue), 0) + self.assertIsNone(self.queue.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + self.queue.push(req1) + self.queue.push(req2) + self.queue.push(req3) + self.assertEqual(len(self.queue), 3) + self.assertEqual(self.queue.pop().url, req1.url) + self.assertEqual(len(self.queue), 2) + self.assertEqual(self.queue.pop().url, req2.url) + self.assertEqual(len(self.queue), 1) + self.assertEqual(self.queue.pop().url, req3.url) + self.assertEqual(len(self.queue), 0) + self.assertIsNone(self.queue.pop()) + + def test_no_peek_raises(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + self.queue.push(Request("https://example.org")) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + self.queue.peek() + + def test_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") + self.assertEqual(len(self.queue), 0) + req1 = Request("https://example.org/1") + req2 = Request("https://example.org/2") + req3 = Request("https://example.org/3") + self.queue.push(req1) + self.queue.push(req2) + self.queue.push(req3) + self.assertEqual(len(self.queue), 3) + self.assertEqual(self.queue.peek().url, req1.url) + self.assertEqual(self.queue.pop().url, req1.url) + self.assertEqual(len(self.queue), 2) + self.assertEqual(self.queue.peek().url, req2.url) + self.assertEqual(self.queue.pop().url, req2.url) + self.assertEqual(len(self.queue), 1) + self.assertEqual(self.queue.peek().url, req3.url) + self.assertEqual(self.queue.pop().url, req3.url) + self.assertIsNone(self.queue.peek()) diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py new file mode 100644 index 000000000..c5fcc1853 --- /dev/null +++ b/tests/test_squeues_request.py @@ -0,0 +1,214 @@ +import shutil +import tempfile +import unittest + +import queuelib + +from scrapy.squeues import ( + PickleFifoDiskQueue, + PickleLifoDiskQueue, + MarshalFifoDiskQueue, + MarshalLifoDiskQueue, + FifoMemoryQueue, + LifoMemoryQueue, +) +from scrapy.http import Request +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler + + +""" +Queues that handle requests +""" + + +class BaseQueueTestCase(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") + self.qpath = self.tempfilename() + self.qdir = self.mkdtemp() + self.crawler = get_crawler(Spider) + + def tearDown(self): + shutil.rmtree(self.tmpdir) + + def tempfilename(self): + with tempfile.NamedTemporaryFile(dir=self.tmpdir) as nf: + return nf.name + + def mkdtemp(self): + return tempfile.mkdtemp(dir=self.tmpdir) + + +class RequestQueueTestMixin: + def queue(self): + raise NotImplementedError() + + def test_one_element_with_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + req = Request("http://www.example.com") + q.push(req) + self.assertEqual(len(q), 1) + self.assertEqual(q.peek().url, req.url) + self.assertEqual(q.pop().url, req.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + q.close() + + def test_one_element_without_peek(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + req = Request("http://www.example.com") + q.push(req) + self.assertEqual(len(q), 1) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + q.peek() + self.assertEqual(q.pop().url, req.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + q.close() + + +class FifoQueueMixin(RequestQueueTestMixin): + def test_fifo_with_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + self.assertEqual(len(q), 3) + self.assertEqual(q.peek().url, req1.url) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.peek().url, req2.url) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.peek().url, req3.url) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + q.close() + + def test_fifo_without_peek(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + q.peek() + self.assertEqual(len(q), 3) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + q.close() + + +class LifoQueueMixin(RequestQueueTestMixin): + def test_lifo_with_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + self.assertEqual(len(q), 3) + self.assertEqual(q.peek().url, req3.url) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.peek().url, req2.url) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.peek().url, req1.url) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + q.close() + + def test_lifo_without_peek(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + q.peek() + self.assertEqual(len(q), 3) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + q.close() + + +class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): + def queue(self): + return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo") + + +class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): + def queue(self): + return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo") + + +class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): + def queue(self): + return MarshalFifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/fifo") + + +class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): + def queue(self): + return MarshalLifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/lifo") + + +class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): + def queue(self): + return FifoMemoryQueue.from_crawler(crawler=self.crawler) + + +class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): + def queue(self): + return LifoMemoryQueue.from_crawler(crawler=self.crawler) From ddea6b7bfa38bf5402d78350ab61e2c827ca49b5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:16:14 -0300 Subject: [PATCH 529/568] Scheduler: minimal interface, API docs (#3559) --- docs/index.rst | 4 + docs/topics/architecture.rst | 5 +- docs/topics/scheduler.rst | 34 ++++ docs/topics/settings.rst | 3 +- scrapy/core/engine.py | 21 ++- scrapy/core/scheduler.py | 298 +++++++++++++++++++++++++++-------- scrapy/utils/job.py | 5 +- tests/test_scheduler_base.py | 159 +++++++++++++++++++ 8 files changed, 458 insertions(+), 71 deletions(-) create mode 100644 docs/topics/scheduler.rst create mode 100644 tests/test_scheduler_base.py diff --git a/docs/index.rst b/docs/index.rst index da264fb34..433798aa8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -227,6 +227,7 @@ Extending Scrapy topics/extensions topics/api topics/signals + topics/scheduler topics/exporters @@ -248,6 +249,9 @@ Extending Scrapy :doc:`topics/signals` See all available signals and how to work with them. +:doc:`topics/scheduler` + Understand the scheduler component. + :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 074c59241..71d027c86 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -87,8 +87,9 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The Scheduler receives requests from the engine and enqueues them for feeding -them later (also to the engine) when the engine requests them. +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine +requests them. .. _component-downloader: diff --git a/docs/topics/scheduler.rst b/docs/topics/scheduler.rst new file mode 100644 index 000000000..57c24b76a --- /dev/null +++ b/docs/topics/scheduler.rst @@ -0,0 +1,34 @@ +.. _topics-scheduler: + +========= +Scheduler +========= + +.. module:: scrapy.core.scheduler + +The scheduler component receives requests from the :ref:`engine ` +and stores them into persistent and/or non-persistent data structures. +It also gets those requests and feeds them back to the engine when it +asks for a next request to be downloaded. + + +Overriding the default scheduler +================================ + +You can use your own custom scheduler class by supplying its full +Python path in the :setting:`SCHEDULER` setting. + + +Minimal scheduler interface +=========================== + +.. autoclass:: BaseScheduler + :members: + + +Default Scrapy scheduler +======================== + +.. autoclass:: Scheduler + :members: + :special-members: __len__ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1d5babcec..e4fb2baf7 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1280,7 +1280,8 @@ SCHEDULER Default: ``'scrapy.core.scheduler.Scheduler'`` -The scheduler to use for crawling. +The scheduler class to be used for crawling. +See the :ref:`topics-scheduler` topic for details. .. setting:: SCHEDULER_DEBUG diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7a09bafa1..dd3225082 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -17,6 +17,7 @@ from scrapy import signals from scrapy.core.scraper import Scraper from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning from scrapy.http import Response, Request +from scrapy.settings import BaseSettings from scrapy.spiders import Spider from scrapy.utils.log import logformatter_adapter, failure_to_exc_info from scrapy.utils.misc import create_instance, load_object @@ -73,12 +74,22 @@ class ExecutionEngine: self.spider: Optional[Spider] = None self.running = False self.paused = False - self.scheduler_cls = load_object(crawler.settings["SCHEDULER"]) + self.scheduler_cls = self._get_scheduler_class(crawler.settings) downloader_cls = load_object(self.settings['DOWNLOADER']) self.downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback = spider_closed_callback + def _get_scheduler_class(self, settings: BaseSettings) -> type: + from scrapy.core.scheduler import BaseScheduler + scheduler_cls = load_object(settings["SCHEDULER"]) + if not issubclass(scheduler_cls, BaseScheduler): + raise TypeError( + f"The provided scheduler class ({settings['SCHEDULER']})" + " does not fully implement the scheduler interface" + ) + return scheduler_cls + @inlineCallbacks def start(self) -> Deferred: if self.running: @@ -301,7 +312,8 @@ class ExecutionEngine: start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider) self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider - yield scheduler.open(spider) + if hasattr(scheduler, "open"): + yield scheduler.open(spider) yield self.scraper.open_spider(spider) self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) @@ -345,8 +357,9 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.scraper.close_spider(spider)) dfd.addErrback(log_failure('Scraper close failure')) - dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) - dfd.addErrback(log_failure('Scheduler close failure')) + if hasattr(self.slot.scheduler, "close"): + dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) + dfd.addErrback(log_failure("Scheduler close failure")) dfd.addBoth(lambda _: self.signals.send_catch_log_deferred( signal=signals.spider_closed, spider=spider, reason=reason, diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 9ce823dbc..5ba0fb63b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,42 +1,179 @@ -import os import json import logging -from os.path import join, exists +import os +from abc import abstractmethod +from os.path import exists, join +from typing import Optional, Type, TypeVar -from scrapy.utils.misc import load_object, create_instance +from twisted.internet.defer import Deferred + +from scrapy.crawler import Crawler +from scrapy.http.request import Request +from scrapy.spiders import Spider from scrapy.utils.job import job_dir +from scrapy.utils.misc import create_instance, load_object logger = logging.getLogger(__name__) -class Scheduler: +class BaseSchedulerMeta(type): """ - Scrapy Scheduler. It allows to enqueue requests and then get - a next request to download. Scheduler is also handling duplication - filtering, via dupefilter. - - Prioritization and queueing is not performed by the Scheduler. - User sets ``priority`` field for each Request, and a PriorityQueue - (defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities - to dequeue requests in a desired order. - - Scheduler uses two PriorityQueue instances, configured to work in-memory - and on-disk (optional). When on-disk queue is present, it is used by - default, and an in-memory queue is used as a fallback for cases where - a disk queue can't handle a request (can't serialize it). - - :setting:`SCHEDULER_MEMORY_QUEUE` and - :setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes - which PriorityQueue instances would be instantiated with, to keep requests - on disk and in memory respectively. - - Overall, Scheduler is an object which holds several PriorityQueue instances - (in-memory and on-disk) and implements fallback logic for them. - Also, it handles dupefilters. + Metaclass to check scheduler classes against the necessary interface """ - def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, - logunser=False, stats=None, pqclass=None, crawler=None): + def __instancecheck__(cls, instance): + return cls.__subclasscheck__(type(instance)) + + def __subclasscheck__(cls, subclass): + return ( + hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests) + and hasattr(subclass, "enqueue_request") and callable(subclass.enqueue_request) + and hasattr(subclass, "next_request") and callable(subclass.next_request) + ) + + +class BaseScheduler(metaclass=BaseSchedulerMeta): + """ + The scheduler component is responsible for storing requests received from + the engine, and feeding them back upon request (also to the engine). + + The original sources of said requests are: + + * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks + * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods + * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods + + The order in which the scheduler returns its stored requests (via the ``next_request`` method) + plays a great part in determining the order in which those requests are downloaded. + + The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with. + """ + + @classmethod + def from_crawler(cls, crawler: Crawler): + """ + Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument. + """ + return cls() + + def open(self, spider: Spider) -> Optional[Deferred]: + """ + Called when the spider is opened by the engine. It receives the spider + instance as argument and it's useful to execute initialization code. + + :param spider: the spider object for the current crawl + :type spider: :class:`~scrapy.spiders.Spider` + """ + pass + + def close(self, reason: str) -> Optional[Deferred]: + """ + Called when the spider is closed by the engine. It receives the reason why the crawl + finished as argument and it's useful to execute cleaning code. + + :param reason: a string which describes the reason why the spider was closed + :type reason: :class:`str` + """ + pass + + @abstractmethod + def has_pending_requests(self) -> bool: + """ + ``True`` if the scheduler has enqueued requests, ``False`` otherwise + """ + raise NotImplementedError() + + @abstractmethod + def enqueue_request(self, request: Request) -> bool: + """ + Process a request received by the engine. + + Return ``True`` if the request is stored correctly, ``False`` otherwise. + + If ``False``, the engine will fire a ``request_dropped`` signal, and + will not make further attempts to schedule the request at a later time. + For reference, the default Scrapy scheduler returns ``False`` when the + request is rejected by the dupefilter. + """ + raise NotImplementedError() + + @abstractmethod + def next_request(self) -> Optional[Request]: + """ + Return the next :class:`~scrapy.http.Request` to be processed, or ``None`` + to indicate that there are no requests to be considered ready at the moment. + + Returning ``None`` implies that no request from the scheduler will be sent + to the downloader in the current reactor cycle. The engine will continue + calling ``next_request`` until ``has_pending_requests`` is ``False``. + """ + raise NotImplementedError() + + +SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler") + + +class Scheduler(BaseScheduler): + """ + Default Scrapy scheduler. This implementation also handles duplication + filtering via the :setting:`dupefilter `. + + This scheduler stores requests into several priority queues (defined by the + :setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues + are backed by either memory or disk based queues (respectively defined by the + :setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings). + + Request prioritization is almost entirely delegated to the priority queue. The only + prioritization performed by this scheduler is using the disk-based queue if present + (i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based + queue if a serialization error occurs. If the disk queue is not present, the memory one + is used directly. + + :param dupefilter: An object responsible for checking and filtering duplicate requests. + The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. + :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: + any class that implements the `BaseDupeFilter` interface + + :param jobdir: The path of a directory to be used for persisting the crawl's state. + The value for the :setting:`JOBDIR` setting is used by default. + See :ref:`topics-jobs`. + :type jobdir: :class:`str` or ``None`` + + :param dqclass: A class to be used as persistent request queue. + The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. + :type dqclass: class + + :param mqclass: A class to be used as non-persistent request queue. + The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. + :type mqclass: class + + :param logunser: A boolean that indicates whether or not unserializable requests should be logged. + The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. + :type logunser: bool + + :param stats: A stats collector object to record stats about the request scheduling process. + The value for the :setting:`STATS_CLASS` setting is used by default. + :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: + any class that implements the `StatsCollector` interface + + :param pqclass: A class to be used as priority queue for requests. + The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. + :type pqclass: class + + :param crawler: The crawler object corresponding to the current crawl. + :type crawler: :class:`scrapy.crawler.Crawler` + """ + def __init__( + self, + dupefilter, + jobdir: Optional[str] = None, + dqclass=None, + mqclass=None, + logunser: bool = False, + stats=None, + pqclass=None, + crawler: Optional[Crawler] = None, + ): self.df = dupefilter self.dqdir = self._dqdir(jobdir) self.pqclass = pqclass @@ -47,34 +184,57 @@ class Scheduler: self.crawler = crawler @classmethod - def from_crawler(cls, crawler): - settings = crawler.settings - dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) - dupefilter = create_instance(dupefilter_cls, settings, crawler) - pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) - dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) - mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) - logunser = settings.getbool('SCHEDULER_DEBUG') - return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, - stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, - mqclass=mqclass, crawler=crawler) + def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV: + """ + Factory method, initializes the scheduler with arguments taken from the crawl settings + """ + dupefilter_cls = load_object(crawler.settings['DUPEFILTER_CLASS']) + return cls( + dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler), + jobdir=job_dir(crawler.settings), + dqclass=load_object(crawler.settings['SCHEDULER_DISK_QUEUE']), + mqclass=load_object(crawler.settings['SCHEDULER_MEMORY_QUEUE']), + logunser=crawler.settings.getbool('SCHEDULER_DEBUG'), + stats=crawler.stats, + pqclass=load_object(crawler.settings['SCHEDULER_PRIORITY_QUEUE']), + crawler=crawler, + ) - def has_pending_requests(self): + def has_pending_requests(self) -> bool: return len(self) > 0 - def open(self, spider): + def open(self, spider: Spider) -> Optional[Deferred]: + """ + (1) initialize the memory queue + (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory + (3) return the result of the dupefilter's ``open`` method + """ self.spider = spider self.mqs = self._mq() self.dqs = self._dq() if self.dqdir else None return self.df.open() - def close(self, reason): - if self.dqs: + def close(self, reason: str) -> Optional[Deferred]: + """ + (1) dump pending requests to disk if there is a disk queue + (2) return the result of the dupefilter's ``close`` method + """ + if self.dqs is not None: state = self.dqs.close() + assert isinstance(self.dqdir, str) self._write_dqs_state(self.dqdir, state) return self.df.close(reason) - def enqueue_request(self, request): + def enqueue_request(self, request: Request) -> bool: + """ + Unless the received request is filtered out by the Dupefilter, attempt to push + it into the disk queue, falling back to pushing it into the memory queue. + + Increment the appropriate stats, such as: ``scheduler/enqueued``, + ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``. + + Return ``True`` if the request was stored successfully, ``False`` otherwise. + """ if not request.dont_filter and self.df.request_seen(request): self.df.log(request, self.spider) return False @@ -87,24 +247,35 @@ class Scheduler: self.stats.inc_value('scheduler/enqueued', spider=self.spider) return True - def next_request(self): + def next_request(self) -> Optional[Request]: + """ + Return a :class:`~scrapy.http.Request` object from the memory queue, + falling back to the disk queue if the memory queue is empty. + Return ``None`` if there are no more enqueued requests. + + Increment the appropriate stats, such as: ``scheduler/dequeued``, + ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. + """ request = self.mqs.pop() - if request: + if request is not None: self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider) else: request = self._dqpop() - if request: + if request is not None: self.stats.inc_value('scheduler/dequeued/disk', spider=self.spider) - if request: + if request is not None: self.stats.inc_value('scheduler/dequeued', spider=self.spider) return request - def __len__(self): - return len(self.dqs) + len(self.mqs) if self.dqs else len(self.mqs) + def __len__(self) -> int: + """ + Return the total amount of enqueued requests + """ + return len(self.dqs) + len(self.mqs) if self.dqs is not None else len(self.mqs) - def _dqpush(self, request): + def _dqpush(self, request: Request) -> bool: if self.dqs is None: - return + return False try: self.dqs.push(request) except ValueError as e: # non serializable request @@ -115,18 +286,18 @@ class Scheduler: logger.warning(msg, {'request': request, 'reason': e}, exc_info=True, extra={'spider': self.spider}) self.logunser = False - self.stats.inc_value('scheduler/unserializable', - spider=self.spider) - return + self.stats.inc_value('scheduler/unserializable', spider=self.spider) + return False else: return True - def _mqpush(self, request): + def _mqpush(self, request: Request) -> None: self.mqs.push(request) - def _dqpop(self): - if self.dqs: + def _dqpop(self) -> Optional[Request]: + if self.dqs is not None: return self.dqs.pop() + return None def _mq(self): """ Create a new priority queue instance, with in-memory storage """ @@ -150,21 +321,22 @@ class Scheduler: {'queuesize': len(q)}, extra={'spider': self.spider}) return q - def _dqdir(self, jobdir): + def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: """ Return a folder name to keep disk queue state at """ - if jobdir: + if jobdir is not None: dqdir = join(jobdir, 'requests.queue') if not exists(dqdir): os.makedirs(dqdir) return dqdir + return None - def _read_dqs_state(self, dqdir): + def _read_dqs_state(self, dqdir: str) -> list: path = join(dqdir, 'active.json') if not exists(path): - return () + return [] with open(path) as f: return json.load(f) - def _write_dqs_state(self, dqdir, state): + def _write_dqs_state(self, dqdir: str, state: list) -> None: with open(join(dqdir, 'active.json'), 'w') as f: json.dump(state, f) diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 4f1e601fc..c92ef36f5 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,7 +1,10 @@ import os +from typing import Optional + +from scrapy.settings import BaseSettings -def job_dir(settings): +def job_dir(settings: BaseSettings) -> Optional[str]: path = settings['JOBDIR'] if path and not os.path.exists(path): os.makedirs(path) diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py new file mode 100644 index 000000000..bf90b4320 --- /dev/null +++ b/tests/test_scheduler_base.py @@ -0,0 +1,159 @@ +from typing import Dict, Optional +from unittest import TestCase +from urllib.parse import urljoin, urlparse + +from testfixtures import LogCapture +from twisted.internet import defer +from twisted.trial.unittest import TestCase as TwistedTestCase + +from scrapy.core.scheduler import BaseScheduler +from scrapy.crawler import CrawlerRunner +from scrapy.http import Request +from scrapy.spiders import Spider +from scrapy.utils.request import request_fingerprint + +from tests.mockserver import MockServer + + +PATHS = ["/a", "/b", "/c"] +URLS = [urljoin("https://example.org", p) for p in PATHS] + + +class MinimalScheduler: + def __init__(self) -> None: + self.requests: Dict[str, Request] = {} + + def has_pending_requests(self) -> bool: + return bool(self.requests) + + def enqueue_request(self, request: Request) -> bool: + fp = request_fingerprint(request) + if fp not in self.requests: + self.requests[fp] = request + return True + return False + + def next_request(self) -> Optional[Request]: + if self.has_pending_requests(): + fp, request = self.requests.popitem() + return request + return None + + +class SimpleScheduler(MinimalScheduler): + def open(self, spider: Spider) -> defer.Deferred: + return defer.succeed("open") + + def close(self, reason: str) -> defer.Deferred: + return defer.succeed("close") + + def __len__(self) -> int: + return len(self.requests) + + +class TestSpider(Spider): + name = "test" + + def __init__(self, mockserver, *args, **kwargs): + super().__init__(*args, **kwargs) + self.start_urls = map(mockserver.url, PATHS) + + def parse(self, response): + return {"path": urlparse(response.url).path} + + +class InterfaceCheckMixin: + def test_scheduler_class(self): + self.assertTrue(isinstance(self.scheduler, BaseScheduler)) + self.assertTrue(issubclass(self.scheduler.__class__, BaseScheduler)) + + +class BaseSchedulerTest(TestCase, InterfaceCheckMixin): + def setUp(self): + self.scheduler = BaseScheduler() + + def test_methods(self): + self.assertIsNone(self.scheduler.open(Spider("foo"))) + self.assertIsNone(self.scheduler.close("finished")) + self.assertRaises(NotImplementedError, self.scheduler.has_pending_requests) + self.assertRaises(NotImplementedError, self.scheduler.enqueue_request, Request("https://example.org")) + self.assertRaises(NotImplementedError, self.scheduler.next_request) + + +class MinimalSchedulerTest(TestCase, InterfaceCheckMixin): + def setUp(self): + self.scheduler = MinimalScheduler() + + def test_open_close(self): + with self.assertRaises(AttributeError): + self.scheduler.open(Spider("foo")) + with self.assertRaises(AttributeError): + self.scheduler.close("finished") + + def test_len(self): + with self.assertRaises(AttributeError): + self.scheduler.__len__() + with self.assertRaises(TypeError): + len(self.scheduler) + + def test_enqueue_dequeue(self): + self.assertFalse(self.scheduler.has_pending_requests()) + for url in URLS: + self.assertTrue(self.scheduler.enqueue_request(Request(url))) + self.assertFalse(self.scheduler.enqueue_request(Request(url))) + self.assertTrue(self.scheduler.has_pending_requests) + + dequeued = [] + while self.scheduler.has_pending_requests(): + request = self.scheduler.next_request() + dequeued.append(request.url) + self.assertEqual(set(dequeued), set(URLS)) + self.assertFalse(self.scheduler.has_pending_requests()) + + +class SimpleSchedulerTest(TwistedTestCase, InterfaceCheckMixin): + def setUp(self): + self.scheduler = SimpleScheduler() + + @defer.inlineCallbacks + def test_enqueue_dequeue(self): + open_result = yield self.scheduler.open(Spider("foo")) + self.assertEqual(open_result, "open") + self.assertFalse(self.scheduler.has_pending_requests()) + + for url in URLS: + self.assertTrue(self.scheduler.enqueue_request(Request(url))) + self.assertFalse(self.scheduler.enqueue_request(Request(url))) + + self.assertTrue(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), len(URLS)) + + dequeued = [] + while self.scheduler.has_pending_requests(): + request = self.scheduler.next_request() + dequeued.append(request.url) + self.assertEqual(set(dequeued), set(URLS)) + + self.assertFalse(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), 0) + + close_result = yield self.scheduler.close("") + self.assertEqual(close_result, "close") + + +class MinimalSchedulerCrawlTest(TwistedTestCase): + scheduler_cls = MinimalScheduler + + @defer.inlineCallbacks + def test_crawl(self): + with MockServer() as mockserver: + settings = {"SCHEDULER": self.scheduler_cls} + with LogCapture() as log: + yield CrawlerRunner(settings).crawl(TestSpider, mockserver) + for path in PATHS: + self.assertIn(f"{{'path': '{path}'}}", str(log)) + self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log)) + + +class SimpleSchedulerCrawlTest(MinimalSchedulerCrawlTest): + scheduler_cls = SimpleScheduler From 02ae1deaf499e79dc8ececf4d5dcea6daef0ade0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 27 Apr 2021 09:41:44 -0300 Subject: [PATCH 530/568] Deprecate unused squeues (#5117) --- scrapy/squeues.py | 47 +++++++++++++++++++++++++++++++++++-------- tests/test_squeues.py | 16 +++++++-------- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 44898ba08..16f7bf4b6 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -8,6 +8,7 @@ import pickle from queuelib import queue +from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.reqser import request_to_dict, request_from_dict @@ -123,30 +124,60 @@ def _pickle_serialize(obj): raise ValueError(str(e)) from e -PickleFifoDiskQueueNonRequest = _serializable_queue( +_PickleFifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads ) -PickleLifoDiskQueueNonRequest = _serializable_queue( +_PickleLifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads ) -MarshalFifoDiskQueueNonRequest = _serializable_queue( +_MarshalFifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads ) -MarshalLifoDiskQueueNonRequest = _serializable_queue( +_MarshalLifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads ) -PickleFifoDiskQueue = _scrapy_serialization_queue(PickleFifoDiskQueueNonRequest) -PickleLifoDiskQueue = _scrapy_serialization_queue(PickleLifoDiskQueueNonRequest) -MarshalFifoDiskQueue = _scrapy_serialization_queue(MarshalFifoDiskQueueNonRequest) -MarshalLifoDiskQueue = _scrapy_serialization_queue(MarshalLifoDiskQueueNonRequest) +# public queue classes +PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue) +PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue) +MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue) +MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) + + +# deprecated queue classes +_subclass_warn_message = "{cls} inherits from deprecated class {old}" +_instance_warn_message = "{cls} is deprecated" +PickleFifoDiskQueueNonRequest = create_deprecated_class( + name="PickleFifoDiskQueueNonRequest", + new_class=_PickleFifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) +PickleLifoDiskQueueNonRequest = create_deprecated_class( + name="PickleLifoDiskQueueNonRequest", + new_class=_PickleLifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) +MarshalFifoDiskQueueNonRequest = create_deprecated_class( + name="MarshalFifoDiskQueueNonRequest", + new_class=_MarshalFifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) +MarshalLifoDiskQueueNonRequest = create_deprecated_class( + name="MarshalLifoDiskQueueNonRequest", + new_class=_MarshalLifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index becacce62..acc821b83 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -3,10 +3,10 @@ import sys from queuelib.tests import test_queue as t from scrapy.squeues import ( - MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue, - MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue, - PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue, - PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue + _MarshalFifoSerializationDiskQueue, + _MarshalLifoSerializationDiskQueue, + _PickleFifoSerializationDiskQueue, + _PickleLifoSerializationDiskQueue, ) from scrapy.item import Item, Field from scrapy.http import Request @@ -53,7 +53,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): chunksize = 100000 def queue(self): - return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize) + return _MarshalFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize) class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): @@ -77,7 +77,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): chunksize = 100000 def queue(self): - return PickleFifoDiskQueue(self.qpath, chunksize=self.chunksize) + return _PickleFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize) def test_serialize_item(self): q = self.queue() @@ -155,13 +155,13 @@ class LifoDiskQueueTestMixin: class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def queue(self): - return MarshalLifoDiskQueue(self.qpath) + return _MarshalLifoSerializationDiskQueue(self.qpath) class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def queue(self): - return PickleLifoDiskQueue(self.qpath) + return _PickleLifoSerializationDiskQueue(self.qpath) def test_serialize_item(self): q = self.queue() From 4f500342c8ad4674b191e1fab0d1b2ac944d7d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Wed, 28 Apr 2021 11:57:44 +0200 Subject: [PATCH 531/568] Require setuptools, scrapy/cmdline.py, /setup.py and tests/test_webclient.py import pkg_resources --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 2b60a10af..b1bb64575 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.0,<4.0', + 'setuptools', ] extras_require = {} cpython_dependencies = [ From 19c7415aae1678631d5ca115a13094b6bd70f245 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 1 May 2021 16:34:39 -0300 Subject: [PATCH 532/568] Request type hints --- scrapy/downloadermiddlewares/retry.py | 2 +- scrapy/http/request/__init__.py | 69 ++++++++++++++++----------- scrapy/utils/curl.py | 2 +- 3 files changed, 42 insertions(+), 31 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5965a1c6c..f1fdc3858 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -98,7 +98,7 @@ def get_retry_request( {'request': request, 'retry_times': retry_times, 'reason': reason}, extra={'spider': spider} ) - new_request = request.copy() + new_request: Request = request.copy() new_request.meta['retry_times'] = retry_times new_request.dont_filter = True if priority_adjust is None: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 498f1b052..3cce9f501 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,22 +4,38 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +from typing import Callable, List, Optional, Type, TypeVar, Union + from w3lib.url import safe_url_string +from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers +from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax -from scrapy.http.common import obsolete_setter -from scrapy.utils.curl import curl_to_request_kwargs + + +RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") class Request(object_ref): - - def __init__(self, url, callback=None, method='GET', headers=None, body=None, - cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, flags=None, cb_kwargs=None): - + def __init__( + self, + url: str, + callback: Optional[Callable] = None, + method: str = "GET", + headers: Optional[dict] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]]=None, + meta: Optional[dict] = None, + encoding: str = "utf-8", + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict] = None, + ) -> None: self._encoding = encoding # this one has to be set first self.method = str(method).upper() self._set_url(url) @@ -44,23 +60,23 @@ class Request(object_ref): self.flags = [] if flags is None else list(flags) @property - def cb_kwargs(self): + def cb_kwargs(self) -> dict: if self._cb_kwargs is None: self._cb_kwargs = {} return self._cb_kwargs @property - def meta(self): + def meta(self) -> dict: if self._meta is None: self._meta = {} return self._meta - def _get_url(self): + def _get_url(self) -> str: return self._url - def _set_url(self, url): + def _set_url(self, url: str) -> None: if not isinstance(url, str): - raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}') + raise TypeError(f"Request url must be str, got {type(url).__name__}") s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) @@ -74,34 +90,28 @@ class Request(object_ref): url = property(_get_url, obsolete_setter(_set_url, 'url')) - def _get_body(self): + def _get_body(self) -> bytes: return self._body - def _set_body(self, body): - if body is None: - self._body = b'' - else: - self._body = to_bytes(body, self.encoding) + def _set_body(self, body: Optional[Union[str, bytes]]) -> None: + self._body = b"" if body is None else to_bytes(body, self.encoding) body = property(_get_body, obsolete_setter(_set_body, 'body')) @property - def encoding(self): + def encoding(self) -> str: return self._encoding - def __str__(self): + def __str__(self) -> str: return f"<{self.method} {self.url}>" __repr__ = __str__ - def copy(self): - """Return a copy of this Request""" + def copy(self) -> RequestTypeVar: return self.replace() - def replace(self, *args, **kwargs): - """Create a new Request with the same attributes except for those - given new values. - """ + def replace(self, *args, **kwargs) -> RequestTypeVar: + """Create a new Request with the same attributes except for those given new values""" for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']: kwargs.setdefault(x, getattr(self, x)) @@ -109,7 +119,9 @@ class Request(object_ref): return cls(*args, **kwargs) @classmethod - def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs): + def from_curl( + cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, **kwargs + ) -> RequestTypeVar: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same @@ -136,8 +148,7 @@ class Request(object_ref): To translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. - - """ + """ request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options) request_kwargs.update(kwargs) return cls(**request_kwargs) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index d8b3deaa1..74f82ad75 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -54,7 +54,7 @@ def _parse_headers_and_cookies(parsed_args): return headers, cookies -def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): +def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = True) -> dict: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command From 34b216289c31c27ad6256ff95382505a9d84adb3 Mon Sep 17 00:00:00 2001 From: Renne Rocha Date: Thu, 6 May 2021 11:34:05 -0300 Subject: [PATCH 533/568] Update link for reasoning value of URLLENGTH_LIMIT (#5134) --- docs/topics/settings.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e4fb2baf7..2506497e2 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1619,7 +1619,7 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. For more information about -the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html +the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 .. setting:: USER_AGENT @@ -1642,7 +1642,6 @@ case to see how to enable and use them. .. settingslist:: - .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search From cec36a9284641bb7b69a2081ad92cd7d4ee25934 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 10 May 2021 13:00:08 -0300 Subject: [PATCH 534/568] Refactor request to/from dict (#5130) --- docs/topics/request-response.rst | 17 ++- scrapy/http/request/__init__.py | 70 ++++++++++-- scrapy/http/request/json_request.py | 8 ++ scrapy/squeues.py | 8 +- scrapy/utils/reqser.py | 103 +++--------------- scrapy/utils/request.py | 27 ++++- ...t_utils_reqser.py => test_request_dict.py} | 76 +++++++++---- 7 files changed, 183 insertions(+), 126 deletions(-) rename tests/{test_utils_reqser.py => test_request_dict.py} (68%) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 500781c05..73b5a858f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -26,10 +26,6 @@ Request objects .. autoclass:: Request - A :class:`Request` object represents an HTTP request, which is usually - generated in the Spider and executed by the Downloader, and thus generating - a :class:`Response`. - :param url: the URL of this request If the URL is invalid, a :exc:`ValueError` exception is raised. @@ -205,6 +201,8 @@ Request objects ``failure.request.cb_kwargs`` in the request's errback. For more information, see :ref:`errback-cb_kwargs`. + .. autoattribute:: Request.attributes + .. method:: Request.copy() Return a new Request which is a copy of this Request. See also: @@ -220,6 +218,15 @@ Request objects .. automethod:: from_curl + .. automethod:: to_dict + + +Other functions related to requests +----------------------------------- + +.. autofunction:: scrapy.utils.request.request_from_dict + + .. _topics-request-response-ref-request-callback-arguments: Passing additional data to callback functions @@ -642,6 +649,8 @@ dealing with JSON requests. data into JSON format. :type dumps_kwargs: dict + .. autoattribute:: JsonRequest.attributes + JsonRequest usage example ------------------------- diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 498f1b052..ad884feac 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,17 +4,37 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +import inspect +from typing import Optional, Tuple + from w3lib.url import safe_url_string +import scrapy +from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers +from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax -from scrapy.http.common import obsolete_setter -from scrapy.utils.curl import curl_to_request_kwargs class Request(object_ref): + """Represents an HTTP request, which is usually generated in a Spider and + executed by the Downloader, thus generating a :class:`Response`. + """ + + attributes: Tuple[str, ...] = ( + "url", "callback", "method", "headers", "body", + "cookies", "meta", "encoding", "priority", + "dont_filter", "errback", "flags", "cb_kwargs", + ) + """A tuple of :class:`str` objects containing the name of all public + attributes of the class that are also keyword parameters of the + ``__init__`` method. + + Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and + :func:`~scrapy.utils.request.request_from_dict`. + """ def __init__(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, @@ -99,11 +119,8 @@ class Request(object_ref): return self.replace() def replace(self, *args, **kwargs): - """Create a new Request with the same attributes except for those - given new values. - """ - for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', - 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']: + """Create a new Request with the same attributes except for those given new values""" + for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) @@ -136,8 +153,43 @@ class Request(object_ref): To translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. - - """ + """ request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options) request_kwargs.update(kwargs) return cls(**request_kwargs) + + def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict: + """Return a dictionary containing the Request's data. + + Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. + + If a spider is given, this method will try to find out the name of the spider methods used as callback + and errback and include them in the output dict, raising an exception if they cannot be found. + """ + d = { + "url": self.url, # urls are safe (safe_string_url) + "callback": _find_method(spider, self.callback) if callable(self.callback) else self.callback, + "errback": _find_method(spider, self.errback) if callable(self.errback) else self.errback, + "headers": dict(self.headers), + } + for attr in self.attributes: + d.setdefault(attr, getattr(self, attr)) + if type(self) is not Request: + d["_class"] = self.__module__ + '.' + self.__class__.__name__ + return d + + +def _find_method(obj, func): + """Helper function for Request.to_dict""" + # Only instance methods contain ``__func__`` + if obj and hasattr(func, '__func__'): + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original function object because instance + # method objects are generated each time attribute is retrieved from instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name + raise ValueError(f"Function {func} is not an instance method in: {obj}") diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index eae3f9f6b..04e80d897 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,12 +8,16 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings +from typing import Tuple from scrapy.http.request import Request from scrapy.utils.deprecate import create_deprecated_class class JsonRequest(Request): + + attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) + def __init__(self, *args, **kwargs): dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) dumps_kwargs.setdefault('sort_keys', True) @@ -36,6 +40,10 @@ class JsonRequest(Request): self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') + @property + def dumps_kwargs(self): + return self._dumps_kwargs + def replace(self, *args, **kwargs): body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 16f7bf4b6..dff9b1350 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -9,7 +9,7 @@ import pickle from queuelib import queue from scrapy.utils.deprecate import create_deprecated_class -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy.utils.request import request_from_dict def _with_mkdir(queue_class): @@ -68,14 +68,14 @@ def _scrapy_serialization_queue(queue_class): return cls(crawler, key) def push(self, request): - request = request_to_dict(request, self.spider) + request = request.to_dict(spider=self.spider) return super().push(request) def pop(self): request = super().pop() if not request: return None - return request_from_dict(request, self.spider) + return request_from_dict(request, spider=self.spider) def peek(self): """Returns the next object to be returned by :meth:`pop`, @@ -87,7 +87,7 @@ def _scrapy_serialization_queue(queue_class): request = super().peek() if not request: return None - return request_from_dict(request, self.spider) + return request_from_dict(request, spider=self.spider) return ScrapyRequestQueue diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index d38b1bc4d..c254b9f82 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -1,95 +1,22 @@ -""" -Helper functions for serializing (and deserializing) requests. -""" -import inspect +import warnings +from typing import Optional -from scrapy.http import Request -from scrapy.utils.python import to_unicode -from scrapy.utils.misc import load_object +import scrapy +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.request import request_from_dict as _from_dict -def request_to_dict(request, spider=None): - """Convert Request object to a dict. - - If a spider is given, it will try to find out the name of the spider method - used in the callback and store that as the callback. - """ - cb = request.callback - if callable(cb): - cb = _find_method(spider, cb) - eb = request.errback - if callable(eb): - eb = _find_method(spider, eb) - d = { - 'url': to_unicode(request.url), # urls should be safe (safe_string_url) - 'callback': cb, - 'errback': eb, - 'method': request.method, - 'headers': dict(request.headers), - 'body': request.body, - 'cookies': request.cookies, - 'meta': request.meta, - '_encoding': request._encoding, - 'priority': request.priority, - 'dont_filter': request.dont_filter, - 'flags': request.flags, - 'cb_kwargs': request.cb_kwargs, - } - if type(request) is not Request: - d['_class'] = request.__module__ + '.' + request.__class__.__name__ - return d +warnings.warn( + ("Module scrapy.utils.reqser is deprecated, please use request.to_dict method" + " and/or scrapy.utils.request.request_from_dict instead"), + category=ScrapyDeprecationWarning, + stacklevel=2, +) -def request_from_dict(d, spider=None): - """Create Request object from a dict. - - If a spider is given, it will try to resolve the callbacks looking at the - spider for methods with the same name. - """ - cb = d['callback'] - if cb and spider: - cb = _get_method(spider, cb) - eb = d['errback'] - if eb and spider: - eb = _get_method(spider, eb) - request_cls = load_object(d['_class']) if '_class' in d else Request - return request_cls( - url=to_unicode(d['url']), - callback=cb, - errback=eb, - method=d['method'], - headers=d['headers'], - body=d['body'], - cookies=d['cookies'], - meta=d['meta'], - encoding=d['_encoding'], - priority=d['priority'], - dont_filter=d['dont_filter'], - flags=d.get('flags'), - cb_kwargs=d.get('cb_kwargs'), - ) +def request_to_dict(request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None) -> dict: + return request.to_dict(spider=spider) -def _find_method(obj, func): - # Only instance methods contain ``__func__`` - if obj and hasattr(func, '__func__'): - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name - raise ValueError(f"Function {func} is not an instance method in: {obj}") - - -def _get_method(obj, name): - name = str(name) - try: - return getattr(obj, name) - except AttributeError: - raise ValueError(f"Method {name!r} not found in: {obj}") +def request_from_dict(d: dict, spider: Optional["scrapy.Spider"] = None) -> "scrapy.Request": + return _from_dict(d, spider=spider) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 541368423..57dcc5f2c 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -11,8 +11,9 @@ from weakref import WeakKeyDictionary from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url -from scrapy.http import Request +from scrapy import Request, Spider from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode @@ -106,3 +107,27 @@ def referer_str(request: Request) -> Optional[str]: if referrer is None: return referrer return to_unicode(referrer, errors='replace') + + +def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: + """Create a :class:`~scrapy.Request` object from a dict. + + If a spider is given, it will try to resolve the callbacks looking at the + spider for methods with the same name. + """ + request_cls = load_object(d["_class"]) if "_class" in d else Request + kwargs = {key: value for key, value in d.items() if key in request_cls.attributes} + if d.get("callback") and spider: + kwargs["callback"] = _get_method(spider, d["callback"]) + if d.get("errback") and spider: + kwargs["errback"] = _get_method(spider, d["errback"]) + return request_cls(**kwargs) + + +def _get_method(obj, name): + """Helper function for request_from_dict""" + name = str(name) + try: + return getattr(obj, name) + except AttributeError: + raise ValueError(f"Method {name!r} not found in: {obj}") diff --git a/tests/test_utils_reqser.py b/tests/test_request_dict.py similarity index 68% rename from tests/test_utils_reqser.py rename to tests/test_request_dict.py index ee68cf6b1..5bdcb975b 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_request_dict.py @@ -1,8 +1,16 @@ +import sys import unittest +import warnings +from contextlib import suppress -from scrapy.http import Request, FormRequest -from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy import Spider, Request +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.http import FormRequest, JsonRequest +from scrapy.utils.request import request_from_dict + + +class CustomRequest(Request): + pass class RequestSerializationTest(unittest.TestCase): @@ -27,7 +35,8 @@ class RequestSerializationTest(unittest.TestCase): priority=20, meta={'a': 'b'}, cb_kwargs={'k': 'v'}, - flags=['testFlag']) + flags=['testFlag'], + ) self._assert_serializes_ok(r, spider=self.spider) def test_latin1_body(self): @@ -39,7 +48,7 @@ class RequestSerializationTest(unittest.TestCase): self._assert_serializes_ok(r) def _assert_serializes_ok(self, request, spider=None): - d = request_to_dict(request, spider=spider) + d = request.to_dict(spider=spider) request2 = request_from_dict(d, spider=spider) self._assert_same_request(request, request2) @@ -54,16 +63,21 @@ class RequestSerializationTest(unittest.TestCase): self.assertEqual(r1.cookies, r2.cookies) self.assertEqual(r1.meta, r2.meta) self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) + self.assertEqual(r1.encoding, r2.encoding) self.assertEqual(r1._encoding, r2._encoding) self.assertEqual(r1.priority, r2.priority) self.assertEqual(r1.dont_filter, r2.dont_filter) self.assertEqual(r1.flags, r2.flags) + if isinstance(r1, JsonRequest): + self.assertEqual(r1.dumps_kwargs, r2.dumps_kwargs) def test_request_class(self): - r = FormRequest("http://www.example.com") - self._assert_serializes_ok(r, spider=self.spider) - r = CustomRequest("http://www.example.com") - self._assert_serializes_ok(r, spider=self.spider) + r1 = FormRequest("http://www.example.com") + self._assert_serializes_ok(r1, spider=self.spider) + r2 = CustomRequest("http://www.example.com") + self._assert_serializes_ok(r2, spider=self.spider) + r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4}) + self._assert_serializes_ok(r3, spider=self.spider) def test_callback_serialization(self): r = Request("http://www.example.com", callback=self.spider.parse_item, @@ -75,7 +89,7 @@ class RequestSerializationTest(unittest.TestCase): callback=self.spider.parse_item_reference, errback=self.spider.handle_error_reference) self._assert_serializes_ok(r, spider=self.spider) - request_dict = request_to_dict(r, self.spider) + request_dict = r.to_dict(spider=self.spider) self.assertEqual(request_dict['callback'], 'parse_item_reference') self.assertEqual(request_dict['errback'], 'handle_error_reference') @@ -84,7 +98,7 @@ class RequestSerializationTest(unittest.TestCase): callback=self.spider._TestSpider__parse_item_reference, errback=self.spider._TestSpider__handle_error_reference) self._assert_serializes_ok(r, spider=self.spider) - request_dict = request_to_dict(r, self.spider) + request_dict = r.to_dict(spider=self.spider) self.assertEqual(request_dict['callback'], '_TestSpider__parse_item_reference') self.assertEqual(request_dict['errback'], @@ -110,18 +124,16 @@ class RequestSerializationTest(unittest.TestCase): def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) - self.assertRaises(ValueError, request_to_dict, r) - self.assertRaises(ValueError, request_to_dict, r, spider=self.spider) + self.assertRaises(ValueError, r.to_dict, spider=self.spider) def test_unserializable_callback2(self): r = Request("http://www.example.com", callback=self.spider.parse_item) - self.assertRaises(ValueError, request_to_dict, r) + self.assertRaises(ValueError, r.to_dict, spider=None) def test_unserializable_callback3(self): """Parser method is removed or replaced dynamically.""" class MySpider(Spider): - name = 'my_spider' def parse(self, response): @@ -130,7 +142,35 @@ class RequestSerializationTest(unittest.TestCase): spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) setattr(spider, 'parse', None) - self.assertRaises(ValueError, request_to_dict, r, spider=spider) + self.assertRaises(ValueError, r.to_dict, spider=spider) + + def test_callback_not_available(self): + """Callback method is not available in the spider passed to from_dict""" + spider = TestSpiderDelegation() + r = Request("http://www.example.com", callback=spider.delegated_callback) + d = r.to_dict(spider=spider) + self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo")) + + +class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest): + def _assert_serializes_ok(self, request, spider=None): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with suppress(KeyError): + del sys.modules["scrapy.utils.reqser"] # delete module to reset the deprecation warning + + from scrapy.utils.reqser import request_from_dict as _from_dict, request_to_dict as _to_dict + + request_copy = _from_dict(_to_dict(request, spider), spider) + self._assert_same_request(request, request_copy) + + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) + self.assertEqual( + "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" + " and/or scrapy.utils.request.request_from_dict instead", + str(caught[0].message), + ) class TestSpiderMixin: @@ -177,7 +217,3 @@ class TestSpider(Spider, TestSpiderMixin): def __parse_item_private(self, response): pass - - -class CustomRequest(Request): - pass From bd60c3f41fd25e7b5a413cf5c112166b813c2691 Mon Sep 17 00:00:00 2001 From: Shinichi Takayanagi Date: Tue, 11 May 2021 04:58:04 +0900 Subject: [PATCH 535/568] More documentation for setting spider atributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: require sphinx-rtd-theme>=0.5.2 and the latest pip to prevent installing breaking docutils>=0.17 * Update feed-exports.rst * Update feed-exports.rst * Reflects the comments * Remove redundant newline * Update docs/topics/feed-exports.rst Co-authored-by: Adrián Chaves * Apply suggestions from code review Co-authored-by: Adrián Chaves Co-authored-by: Adrián Chaves Co-authored-by: Eugenio Lacuesta --- docs/topics/feed-exports.rst | 3 +++ docs/topics/spiders.rst | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index e772a461c..26c247cdd 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -135,6 +135,9 @@ Here are some examples to illustrate: - ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` +.. note:: :ref:`Spider arguments ` become spider attributes, hence + they can also be used as storage URI parameters. + .. _topics-feed-storage-backends: diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 2056664c7..a3e9f410f 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -294,6 +294,14 @@ The above example can also be written as follows:: def start_requests(self): yield scrapy.Request(f'http://www.example.com/categories/{self.category}') +If you are :ref:`running Scrapy from a script `, you can +specify spider arguments when calling +:class:`CrawlerProcess.crawl ` or +:class:`CrawlerRunner.crawl `:: + + process = CrawlerProcess() + process.crawl(MySpider, category="electronics") + Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. If you were to set the ``start_urls`` attribute from the command line, From c5b1ee810167266fcd259f263dbfc0fe0204761a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 May 2021 09:04:53 +0200 Subject: [PATCH 536/568] Make Twisted[http2] installation optional (#5113) Co-authored-by: Eugenio Lacuesta --- conftest.py | 9 +++++++++ docs/topics/settings.rst | 14 ++++++++----- setup.py | 3 +-- tests/test_downloader_handlers_http2.py | 26 ++++++++++++++++++++----- tests/test_http2_client_protocol.py | 13 ++++++++----- tox.ini | 11 +++++------ 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/conftest.py b/conftest.py index e4dd80de0..4931c5a79 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,7 @@ from pathlib import Path import pytest +from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor @@ -25,6 +26,14 @@ for line in open('tests/ignores.txt'): if file_path and file_path[0] != '#': collect_ignore.append(file_path) +if not H2_ENABLED: + collect_ignore.extend( + ( + 'scrapy/core/downloader/handlers/http2.py', + *_py_files("scrapy/core/http2"), + ) + ) + @pytest.fixture() def chdir(tmpdir): diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2506497e2..0b290598f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -680,12 +680,16 @@ handler (without replacement), place this in your ``settings.py``:: .. _http2: -The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update -:setting:`DOWNLOAD_HANDLERS` as follows:: +The default HTTPS handler uses HTTP/1.1. To use HTTP/2: - DOWNLOAD_HANDLERS = { - 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', - } +#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to + enable HTTP/2 support in Twisted. + +#. Update :setting:`DOWNLOAD_HANDLERS` as follows:: + + DOWNLOAD_HANDLERS = { + 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', + } .. warning:: diff --git a/setup.py b/setup.py index b1bb64575..ed2b6e347 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support(): install_requires = [ - 'Twisted[http2]>=17.9.0', + 'Twisted>=17.9.0', 'cryptography>=2.0', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', @@ -31,7 +31,6 @@ install_requires = [ 'zope.interface>=4.1.3', 'protego>=0.1.15', 'itemadapter>=0.1.0', - 'h2>=3.0,<4.0', 'setuptools', ] extras_require = {} diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 439778014..53bb4fe92 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -1,5 +1,5 @@ import json -from unittest import mock +from unittest import mock, skipIf from pytest import mark from testfixtures import LogCapture @@ -7,8 +7,8 @@ from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server from twisted.web.error import SchemeNotSupported +from twisted.web.http import H2_ENABLED -from scrapy.core.downloader.handlers.http2 import H2DownloadHandler from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.misc import create_instance @@ -21,11 +21,17 @@ from tests.test_downloader_handlers import ( ) +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2TestCase(Https11TestCase): + scheme = 'https' - download_handler_cls = H2DownloadHandler HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + @classmethod + def setUpClass(cls): + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler + def test_protocol(self): request = Request(self.getURL("host"), method="GET") d = self.download_request(request, Spider("foo")) @@ -187,9 +193,14 @@ class Https2InvalidDNSPattern(Https2TestCase): super(Https2InvalidDNSPattern, self).setUp() +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2CustomCiphers(Https11CustomCiphers): scheme = 'https' - download_handler_cls = H2DownloadHandler + + @classmethod + def setUpClass(cls): + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler class Http2MockServerTestCase(Http11MockServerTestCase): @@ -201,6 +212,7 @@ class Http2MockServerTestCase(Http11MockServerTestCase): } +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2ProxyTestCase(Http11ProxyTestCase): # only used for HTTPS tests keyfile = 'keys/localhost.key' @@ -209,9 +221,13 @@ class Https2ProxyTestCase(Http11ProxyTestCase): scheme = 'https' host = u'127.0.0.1' - download_handler_cls = H2DownloadHandler expected_http_proxy_request_body = b'/' + @classmethod + def setUpClass(cls): + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler + def setUp(self): site = server.Site(UriResource(), timeout=None) self.port = reactor.listenSSL( diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 8b2f6a11d..677ede92b 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,10 +5,9 @@ import re import shutil import string from ipaddress import IPv4Address -from unittest import mock +from unittest import mock, skipIf from urllib.parse import urlencode -from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint @@ -17,12 +16,10 @@ from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certif from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.client import ResponseFailed, URI -from twisted.web.http import Request as TxRequest +from twisted.web.http import H2_ENABLED, Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File -from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol -from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings from scrapy.spiders import Spider @@ -173,6 +170,7 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: return PrivateCertificate.loadPEM(pem) +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2ClientProtocolTestCase(TestCase): scheme = 'https' key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key') @@ -220,6 +218,7 @@ class Https2ClientProtocolTestCase(TestCase): uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) self.conn_closed_deferred = Deferred() + from scrapy.core.http2.protocol import H2ClientFactory h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) @@ -426,6 +425,7 @@ class Https2ClientProtocolTestCase(TestCase): def assert_failure(failure: Failure): self.assertTrue(len(failure.value.reasons) > 0) + from h2.exceptions import InvalidBodyLengthError self.assertTrue(any( isinstance(error, InvalidBodyLengthError) for error in failure.value.reasons @@ -511,6 +511,7 @@ class Https2ClientProtocolTestCase(TestCase): def assert_inactive_stream(failure): self.assertIsNotNone(failure.check(ResponseFailed)) + from scrapy.core.http2.stream import InactiveStreamClosed self.assertTrue(any( isinstance(e, InactiveStreamClosed) for e in failure.value.reasons @@ -596,6 +597,7 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(url) def assert_invalid_hostname(failure: Failure): + from scrapy.core.http2.stream import InvalidHostname self.assertIsNotNone(failure.check(InvalidHostname)) error_msg = str(failure.value) self.assertIn('localhost', error_msg) @@ -633,6 +635,7 @@ class Https2ClientProtocolTestCase(TestCase): def assert_timeout_error(failure: Failure): for err in failure.value.reasons: + from scrapy.core.http2.protocol import H2ClientProtocol if isinstance(err, TimeoutError): self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err)) break diff --git a/tox.ini b/tox.ini index 5b0606f8f..8167aff96 100644 --- a/tox.ini +++ b/tox.ini @@ -50,6 +50,8 @@ commands = basepython = python3 deps = {[testenv]deps} + # Twisted[http2] is required to import some files + Twisted[http2]>=17.9.0 pytest-flake8 commands = py.test --flake8 {posargs:docs scrapy tests} @@ -57,12 +59,7 @@ commands = [testenv:pylint] basepython = python3 deps = - {[testenv]deps} - # Optional dependencies - boto - reppy - robotexclusionrulesparser - # Test dependencies + {[testenv:extra-deps]deps} pylint commands = pylint conftest.py docs extras scrapy setup.py tests @@ -119,9 +116,11 @@ setenv = [testenv:extra-deps] deps = {[testenv]deps} + boto reppy robotexclusionrulesparser Pillow>=4.0.0 + Twisted[http2]>=17.9.0 [testenv:asyncio] commands = From ee682af3b06d48815dbdaa27c1177b94aaf679e1 Mon Sep 17 00:00:00 2001 From: Bhavesh <35660861+Bhavesh0327@users.noreply.github.com> Date: Wed, 12 May 2021 01:53:02 +0530 Subject: [PATCH 537/568] [Fix] Change the truncation limit of Proxy TunnelError from 32 to 1000 (#5007) * [Fix] Change the truncation limit oof Proxy TunnelError from 32 to 64 * [Fix] Change the truncation limit for Proxy tunnel error * [Fix] flake8 check * [Fix] formatting issues * [Remove] coverage report * [Fix] truncation error issue * [Fix] formatting issues * [Remove] coverage report --- scrapy/core/downloader/handlers/http11.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 25cb3ec62..073f35891 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -98,8 +98,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): with this endpoint comes from the pool and a CONNECT has already been issued for it. """ - - _responseMatcher = re.compile(br'HTTP/1\.. (?P\d{3})(?P.{,32})') + _truncatedLength = 1000 + _responseAnswer = r'HTTP/1\.. (?P\d{3})(?P.{,' + str(_truncatedLength) + r'})' + _responseMatcher = re.compile(_responseAnswer.encode()) def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf @@ -144,7 +145,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): extra = {'status': int(respm.group('status')), 'reason': respm.group('reason').strip()} else: - extra = rcvd_bytes[:32] + extra = rcvd_bytes[:self._truncatedLength] self._tunnelReadyDeferred.errback( TunnelError('Could not open CONNECT tunnel with proxy ' f'{self._host}:{self._port} [{extra!r}]') From 23cfdb058e80a18ee2b66e1b966355f3aca426d0 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 28 May 2021 09:45:06 +0000 Subject: [PATCH 538/568] Reducing amount of warnings during test run (#5162) * put flake8 options into separate file to remove pytest warnings * remove ResourceLeaked warning in pypy * suppress warnings from twisted * ignore deprecation warnings here * ignore deprecation warning in tests of deprecated methods * ignore deprecation warnings here * update test classes * don`t use deprecated method call * ignore deprecation warnings here * proper warning class * more selective ignoring * Revert "don`t use deprecated method call" This reverts commit 59216ab5603c4b47574382768614ef4c39d36747. --- .flake8 | 19 +++++++++++++++++++ .gitignore | 2 ++ conftest.py | 9 +++++---- pytest.ini | 19 ++----------------- tests/test_exporters.py | 12 ++++++++---- tests/test_feedexport.py | 4 ++-- tests/test_http_response.py | 12 ++++++++---- tests/test_item.py | 22 ++++++++++++---------- tests/test_utils_deprecate.py | 2 +- tests/test_utils_python.py | 9 +++++++-- 10 files changed, 66 insertions(+), 44 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..1c503fb0b --- /dev/null +++ b/.flake8 @@ -0,0 +1,19 @@ +[flake8] + +max-line-length = 119 +ignore = W503 + +exclude = +# Exclude files that are meant to provide top-level imports +# E402: Module level import not at top of file +# F401: Module imported but unused + scrapy/__init__.py E402 + scrapy/core/downloader/handlers/http.py F401 + scrapy/http/__init__.py F401 + scrapy/linkextractors/__init__.py E402 F401 + scrapy/selector/__init__.py F401 + scrapy/spiders/__init__.py E402 F401 + + # Issues pending a review: + scrapy/utils/url.py F403 F405 + tests/test_loader.py E741 diff --git a/.gitignore b/.gitignore index 795e2605e..d77d24624 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ htmlcov/ .coverage .pytest_cache/ .coverage.* +coverage.* +test-output.* .cache/ .mypy_cache/ /tests/keys/localhost.crt diff --git a/conftest.py b/conftest.py index 4931c5a79..05b4ccdad 100644 --- a/conftest.py +++ b/conftest.py @@ -21,10 +21,11 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -for line in open('tests/ignores.txt'): - file_path = line.strip() - if file_path and file_path[0] != '#': - collect_ignore.append(file_path) +with open('tests/ignores.txt') as reader: + for line in reader: + file_path = line.strip() + if file_path and file_path[0] != '#': + collect_ignore.append(file_path) if not H2_ENABLED: collect_ignore.extend( diff --git a/pytest.ini b/pytest.ini index 0aae09ff5..6de08c78d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,20 +20,5 @@ addopts = --ignore=docs/utils markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed -flake8-max-line-length = 119 -flake8-ignore = - W503 - - # Exclude files that are meant to provide top-level imports - # E402: Module level import not at top of file - # F401: Module imported but unused - scrapy/__init__.py E402 - scrapy/core/downloader/handlers/http.py F401 - scrapy/http/__init__.py F401 - scrapy/linkextractors/__init__.py E402 F401 - scrapy/selector/__init__.py F401 - scrapy/spiders/__init__.py E402 F401 - - # Issues pending a review: - scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 +filterwarnings= + ignore::DeprecationWarning:twisted.web.test.test_webclient diff --git a/tests/test_exporters.py b/tests/test_exporters.py index ebc477e74..04bae31d3 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -6,12 +6,14 @@ import tempfile import unittest from io import BytesIO from datetime import datetime +from warnings import catch_warnings, filterwarnings import lxml.etree from itemadapter import ItemAdapter from scrapy.item import Item, Field from scrapy.utils.python import to_unicode +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exporters import ( BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, @@ -172,10 +174,12 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_binary(self): - exporter = PythonItemExporter(binary=True) - value = self.item_class(name='John\xa3', age='22') - expected = {b'name': b'John\xc2\xa3', b'age': b'22'} - self.assertEqual(expected, exporter.export_item(value)) + with catch_warnings(): + filterwarnings('ignore', category=ScrapyDeprecationWarning) + exporter = PythonItemExporter(binary=True) + value = self.item_class(name='John\xa3', age='22') + expected = {b'name': b'John\xc2\xa3', b'age': b'22'} + self.assertEqual(expected, exporter.export_item(value)) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d248824fc..df7ec4461 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -515,7 +515,7 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): class DummyBlockingFeedStorage(BlockingFeedStorage): - def __init__(self, uri): + def __init__(self, uri, *args, feed_options=None): self.path = file_uri_to_path(uri) def _store_in_thread(self, file): @@ -541,7 +541,7 @@ class LogOnStoreFileStorage: It can be used to make sure `store` method is invoked. """ - def __init__(self, uri): + def __init__(self, uri, feed_options=None): self.path = file_uri_to_path(uri) self.logger = getLogger() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index f831ef5dc..04a594d03 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,6 +1,6 @@ import unittest from unittest import mock -from warnings import catch_warnings +from warnings import catch_warnings, filterwarnings from w3lib.encoding import resolve_encoding @@ -134,7 +134,9 @@ class BaseResponseTest(unittest.TestCase): assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) - self.assertEqual(response.body_as_unicode(), body_unicode) + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + self.assertEqual(response.body_as_unicode(), body_unicode) self.assertEqual(response.text, body_unicode) def _assert_response_encoding(self, response, encoding): @@ -345,8 +347,10 @@ class TextResponseTest(BaseResponseTest): r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') # check body_as_unicode - self.assertTrue(isinstance(r1.body_as_unicode(), str)) - self.assertEqual(r1.body_as_unicode(), unicode_string) + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + self.assertTrue(isinstance(r1.body_as_unicode(), str)) + self.assertEqual(r1.body_as_unicode(), unicode_string) # check response.text self.assertTrue(isinstance(r1.text, str)) diff --git a/tests/test_item.py b/tests/test_item.py index 78d204e34..c94bb44af 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,6 +1,6 @@ import unittest from unittest import mock -from warnings import catch_warnings +from warnings import catch_warnings, filterwarnings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta @@ -328,16 +328,18 @@ class BaseItemTest(unittest.TestCase): class SubclassedItem(Item): pass - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem)) - self.assertTrue(isinstance(Item(), BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), BaseItem)) + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + self.assertTrue(isinstance(BaseItem(), BaseItem)) + self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem)) + self.assertTrue(isinstance(Item(), BaseItem)) + self.assertTrue(isinstance(SubclassedItem(), BaseItem)) - # make sure internal checks using private _BaseItem class succeed - self.assertTrue(isinstance(BaseItem(), _BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), _BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), _BaseItem)) + # make sure internal checks using private _BaseItem class succeed + self.assertTrue(isinstance(BaseItem(), _BaseItem)) + self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem)) + self.assertTrue(isinstance(Item(), _BaseItem)) + self.assertTrue(isinstance(SubclassedItem(), _BaseItem)) def test_deprecation_warning(self): """ diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 35d35b45d..e47afa266 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -108,7 +108,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): # ignore subclassing warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', ScrapyDeprecationWarning) + warnings.simplefilter('ignore', MyWarning) class UserClass(Deprecated): pass diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3115cc92f..4b3964154 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -5,8 +5,9 @@ import platform import unittest from datetime import datetime from itertools import count -from warnings import catch_warnings +from warnings import catch_warnings, filterwarnings +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, WeakKeyCache, get_func_args, to_bytes, to_unicode, @@ -160,7 +161,11 @@ class UtilsPythonTestCase(unittest.TestCase): pass _values = count() - wk = WeakKeyCache(lambda k: next(_values)) + + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + wk = WeakKeyCache(lambda k: next(_values)) + k = _Weakme() v = wk[k] self.assertEqual(v, wk[k]) From 216dd37953112e360348e19eaf8cae45a52fe87a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 11:16:40 -0300 Subject: [PATCH 539/568] Type hints for FormRequest --- scrapy/http/request/form.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ef2eb3ba6..4465f40ae 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,6 +5,7 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +from typing import Optional, Type, TypeVar from urllib.parse import urljoin, urlencode import lxml.html @@ -12,15 +13,18 @@ from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request +from scrapy.http.response.text import TextResponse from scrapy.utils.python import to_bytes, is_listlike from scrapy.utils.response import get_base_url +FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") + + class FormRequest(Request): valid_form_methods = ['GET', 'POST'] - def __init__(self, *args, **kwargs): - formdata = kwargs.pop('formdata', None) + def __init__(self, *args, formdata: Optional[dict] = None, **kwargs) -> None: if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' @@ -36,9 +40,19 @@ class FormRequest(Request): self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr) @classmethod - def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, - clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs): - + def from_response( + cls: Type[FormRequestTypeVar], + response: TextResponse, + formname: Optional[str] = None, + formid: Optional[str] = None, + formnumber: Optional[int] = 0, + formdata: Optional[dict] = None, + clickdata: Optional[dict] = None, + dont_click: bool = False, + formxpath: Optional[str] = None, + formcss: Optional[str] = None, + **kwargs, + ) -> FormRequestTypeVar: kwargs.setdefault('encoding', response.encoding) if formcss is not None: From c594017e518af31edc24619bfc590907ba6280c7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 11:27:21 -0300 Subject: [PATCH 540/568] Type hints for private functions used by FormRequest --- scrapy/http/request/form.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 4465f40ae..781e3495a 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst from typing import Optional, Type, TypeVar from urllib.parse import urljoin, urlencode -import lxml.html +from lxml.html import HTMLParser, FormElement from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace @@ -72,7 +72,7 @@ class FormRequest(Request): return cls(url=url, method=method, formdata=formdata, **kwargs) -def _get_form_url(form, url): +def _get_form_url(form: FormElement, url: Optional[str]) -> str: if url is None: action = form.get('action') if action is None: @@ -88,10 +88,15 @@ def _urlencode(seq, enc): return urlencode(values, doseq=True) -def _get_form(response, formname, formid, formnumber, formxpath): - """Find the form element """ - root = create_root_node(response.text, lxml.html.HTMLParser, - base_url=get_base_url(response)) +def _get_form( + response: TextResponse, + formname: Optional[str], + formid: Optional[str], + formnumber: Optional[int], + formxpath: Optional[str], +) -> FormElement: + """Find the wanted form element within the given response.""" + root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: raise ValueError(f"No element found in {response}") @@ -119,8 +124,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): break raise ValueError(f'No element found with {formxpath}') - # If we get here, it means that either formname was None - # or invalid + # If we get here, it means that either formname was None or invalid if formnumber is not None: try: form = forms[formnumber] From 85f88a5710e51a3137ded72f5fdb1c3465480355 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 12:02:16 -0300 Subject: [PATCH 541/568] More type hints for private functions used by FormRequest --- scrapy/http/request/form.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 781e3495a..1ad878c03 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,7 +5,7 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import Optional, Type, TypeVar +from typing import List, Optional, Tuple, Type, TypeVar, Union from urllib.parse import urljoin, urlencode from lxml.html import HTMLParser, FormElement @@ -20,11 +20,13 @@ from scrapy.utils.response import get_base_url FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") +FormdataType = Optional[Union[dict, List[Tuple[str, str]]]] + class FormRequest(Request): valid_form_methods = ['GET', 'POST'] - def __init__(self, *args, formdata: Optional[dict] = None, **kwargs) -> None: + def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None: if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' @@ -46,7 +48,7 @@ class FormRequest(Request): formname: Optional[str] = None, formid: Optional[str] = None, formnumber: Optional[int] = 0, - formdata: Optional[dict] = None, + formdata: FormdataType = None, clickdata: Optional[dict] = None, dont_click: bool = False, formxpath: Optional[str] = None, @@ -60,7 +62,7 @@ class FormRequest(Request): formxpath = HTMLTranslator().css_to_xpath(formcss) form = _get_form(response, formname, formid, formnumber, formxpath) - formdata = _get_inputs(form, formdata, dont_click, clickdata, response) + formdata = _get_inputs(form, formdata, dont_click, clickdata) url = _get_form_url(form, kwargs.pop('url', None)) method = kwargs.pop('method', form.method) @@ -134,22 +136,27 @@ def _get_form( return form -def _get_inputs(form, formdata, dont_click, clickdata, response): +def _get_inputs( + form: FormElement, + formdata: FormdataType, + dont_click: bool, + clickdata: Optional[dict], +) -> List[Tuple[str, str]]: + """Return a list of key-value pairs for the inputs found in the given form.""" try: formdata_keys = dict(formdata or ()).keys() except (ValueError, TypeError): raise ValueError('formdata should be a dict or iterable of tuples') if not formdata: - formdata = () + formdata = [] inputs = form.xpath('descendant::textarea' '|descendant::select' '|descendant::input[not(@type) or @type[' ' not(re:test(., "^(?:submit|image|reset)$", "i"))' ' and (../@checked or' ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', - namespaces={ - "re": "http://exslt.org/regular-expressions"}) + namespaces={"re": "http://exslt.org/regular-expressions"}) values = [(k, '' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys] @@ -160,7 +167,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): values.append(clickable) if isinstance(formdata, dict): - formdata = formdata.items() + formdata = formdata.items() # type: ignore[assignment] values.extend((k, v) for k, v in formdata if v is not None) return values @@ -189,7 +196,7 @@ def _select_value(ele, n, v): return n, v -def _get_clickable(clickdata, form): +def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tuple[str, str]]: """ Returns the clickable element specified in clickdata, if the latter is given. If not, it returns the first @@ -201,7 +208,7 @@ def _get_clickable(clickdata, form): namespaces={"re": "http://exslt.org/regular-expressions"} )) if not clickables: - return + return None # If we don't have clickdata, we just use the first clickable element if clickdata is None: From c9fecca010a3ddddd1145f65a78dde30b5c71a72 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 12:25:26 -0300 Subject: [PATCH 542/568] More type hints --- scrapy/http/request/form.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 1ad878c03..c3e112041 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,10 +5,10 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import List, Optional, Tuple, Type, TypeVar, Union +from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union from urllib.parse import urljoin, urlencode -from lxml.html import HTMLParser, FormElement +from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace @@ -83,7 +83,7 @@ def _get_form_url(form: FormElement, url: Optional[str]) -> str: return urljoin(form.base_url, url) -def _urlencode(seq, enc): +def _urlencode(seq: Iterable, enc: str) -> str: values = [(to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq for v in (vs if is_listlike(vs) else [vs])] @@ -157,9 +157,11 @@ def _get_inputs( ' and (../@checked or' ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={"re": "http://exslt.org/regular-expressions"}) - values = [(k, '' if v is None else v) - for k, v in (_value(e) for e in inputs) - if k and k not in formdata_keys] + values = [ + (k, '' if v is None else v) + for k, v in (_value(e) for e in inputs) + if k and k not in formdata_keys + ] if not dont_click: clickable = _get_clickable(clickdata, form) @@ -173,7 +175,7 @@ def _get_inputs( return values -def _value(ele): +def _value(ele: HtmlElement): n = ele.name v = ele.value if ele.tag == 'select': @@ -181,7 +183,7 @@ def _value(ele): return n, v -def _select_value(ele, n, v): +def _select_value(ele: SelectElement, n: str, v: str): multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected @@ -192,7 +194,8 @@ def _select_value(ele, n, v): # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') - v = [(o.get('value') or o.text or '').strip() for o in selected_options] + values = [(o.get('value') or o.text or '').strip() for o in selected_options] + return n, values return n, v From 479260dca012b3d03221f2e6322008448d0fb8b5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 12:52:46 -0300 Subject: [PATCH 543/568] Type hints for Request subclasses --- scrapy/http/request/json_request.py | 17 +++++++---------- scrapy/http/request/rpc.py | 4 ++-- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 04e80d897..dba3c3a82 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,9 +8,9 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings -from typing import Tuple +from typing import Optional, Tuple -from scrapy.http.request import Request +from scrapy.http.request import Request, RequestTypeVar from scrapy.utils.deprecate import create_deprecated_class @@ -18,8 +18,8 @@ class JsonRequest(Request): attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) - def __init__(self, *args, **kwargs): - dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) + def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None: + dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault('sort_keys', True) self._dumps_kwargs = dumps_kwargs @@ -29,10 +29,8 @@ class JsonRequest(Request): if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') - elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) - if 'method' not in kwargs: kwargs['method'] = 'POST' @@ -41,23 +39,22 @@ class JsonRequest(Request): self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') @property - def dumps_kwargs(self): + def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args, **kwargs): + def replace(self, *args, **kwargs) -> RequestTypeVar: body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') - elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) return super().replace(*args, **kwargs) - def _dumps(self, data): + def _dumps(self, data: dict) -> str: """Convert to JSON """ return json.dumps(data, **self._dumps_kwargs) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index c70912e49..06d98cea5 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -5,6 +5,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ import xmlrpc.client as xmlrpclib +from typing import Optional from scrapy.http.request import Request from scrapy.utils.python import get_func_args @@ -15,8 +16,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - def __init__(self, *args, **kwargs): - encoding = kwargs.get('encoding', None) + def __init__(self, *args, encoding: Optional[str] = None, **kwargs): if 'body' not in kwargs and 'params' in kwargs: kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) kwargs['body'] = xmlrpclib.dumps(**kw) From ce6447731a91e32faf17564d21ddc3b88e582f50 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 7 Jun 2021 13:25:04 -0300 Subject: [PATCH 544/568] Replace return type --- scrapy/http/request/__init__.py | 4 ++-- scrapy/http/request/json_request.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 8f00c20b7..7672dec00 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -126,10 +126,10 @@ class Request(object_ref): __repr__ = __str__ - def copy(self) -> RequestTypeVar: + def copy(self) -> "Request": return self.replace() - def replace(self, *args, **kwargs) -> RequestTypeVar: + def replace(self, *args, **kwargs) -> "Request": """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index dba3c3a82..728a2a104 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -10,7 +10,7 @@ import json import warnings from typing import Optional, Tuple -from scrapy.http.request import Request, RequestTypeVar +from scrapy.http.request import Request from scrapy.utils.deprecate import create_deprecated_class @@ -42,7 +42,7 @@ class JsonRequest(Request): def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args, **kwargs) -> RequestTypeVar: + def replace(self, *args, **kwargs) -> Request: body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None From e876d8e38780a88e4c1fc060e3ead779222be83a Mon Sep 17 00:00:00 2001 From: Veniamin Gvozdikov Date: Fri, 11 Jun 2021 09:22:04 +0300 Subject: [PATCH 545/568] Rename scrapy-crawlera to scrapy-zyte-smartproxy (#5074) --- scrapy/core/downloader/handlers/http11.py | 6 +++--- scrapy/core/downloader/handlers/http2.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 073f35891..50486d13c 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -283,9 +283,9 @@ class ScrapyAgent: if omitConnectTunnel: warnings.warn( "Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Zyte Smart Proxy Manager (formerly Crawlera), " - "it doesn't require this mode anymore, so you should " - "update scrapy-crawlera to 1.3.0+ and remove '?noconnect' " + "If you use Zyte Smart Proxy Manager, it doesn't require " + "this mode anymore, so you should update scrapy-crawlera " + "to scrapy-zyte-smartproxy and remove '?noconnect' " "from the Zyte Smart Proxy Manager URL.", ScrapyDeprecationWarning, ) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index e97c31e90..7bb88a193 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -72,10 +72,14 @@ class ScrapyH2Agent: proxy_host = proxy_host.decode() omit_connect_tunnel = b'noconnect' in proxy_params if omit_connect_tunnel: - warnings.warn("Using HTTPS proxies in the noconnect mode is not supported by the " - "downloader handler. If you use Crawlera, it doesn't require this " - "mode anymore, so you should update scrapy-crawlera to 1.3.0+ " - "and remove '?noconnect' from the Crawlera URL.") + warnings.warn( + "Using HTTPS proxies in the noconnect mode is not " + "supported by the downloader handler. If you use Zyte " + "Smart Proxy Manager, it doesn't require this mode " + "anymore, so you should update scrapy-crawlera to " + "scrapy-zyte-smartproxy and remove '?noconnect' from the " + "Zyte Smart Proxy Manager URL." + ) if scheme == b'https' and not omit_connect_tunnel: # ToDo From 28858574d92fd2170575bea83fc6abac6b45b63a Mon Sep 17 00:00:00 2001 From: pdt1931 <56060869+pdt1931@users.noreply.github.com> Date: Fri, 11 Jun 2021 00:49:41 -0700 Subject: [PATCH 546/568] Add FAQ to code of Conduct (#5177) * added to CODE_OF_CONDUCT.md to include link to FAQ about the code of conduct * added to CODE_OF_CONDUCT.md to include link to FAQ about the code of conduct --- CODE_OF_CONDUCT.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 652460383..902cd523e 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -72,3 +72,6 @@ available at [http://contributor-covenant.org/version/1/4][version]. [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq From 66e200423943a921326334a37777720c706ee2b5 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 14 Jun 2021 18:58:35 +0500 Subject: [PATCH 547/568] Fix a flake8 problem --- tests/test_http_request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index ea32c6711..b610087bd 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -381,7 +381,8 @@ class FormRequestTest(RequestTest): def test_formdata_overrides_querystring(self): data = (('a', 'one'), ('a', 'two'), ('b', '2')) - url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] + url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', + method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) From 5044549c550876cc31fb2e15aaa5013e8fc333c3 Mon Sep 17 00:00:00 2001 From: Ajay Mittur Date: Mon, 14 Jun 2021 14:58:19 +0000 Subject: [PATCH 548/568] Update proxyScheme assignment --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 15de8cdbd..d2f9084fc 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -319,7 +319,7 @@ class ScrapyAgent: pool=self._pool, ) else: - proxyScheme = b'http' if not proxyScheme else proxyScheme + proxyScheme = proxyScheme or b'http' proxyHost = to_bytes(proxyHost, encoding='ascii') proxyPort = to_bytes(str(proxyPort), encoding='ascii') proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) From 7d653288e3d9369e4c007d223327f919b99b7737 Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Mon, 14 Jun 2021 21:39:18 +0530 Subject: [PATCH 549/568] Update unittest --- tests/test_downloader_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 67224ed53..9c11820e5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -768,7 +768,7 @@ class Http11ProxyTestCase(HttpProxyTestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'http://example.com') + self.assertEqual(response.body, self.expected_http_proxy_request_body) http_proxy = self.getURL('').replace('http://', '') request = Request('http://example.com', meta={'proxy': http_proxy}) From 812b4bb51855605a36bcfb166f66a121cc67f97c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Wed, 23 Jun 2021 17:09:28 +0100 Subject: [PATCH 550/568] CloseSpider can be raised on spider_idle signal handler --- docs/topics/signals.rst | 3 +++ scrapy/core/engine.py | 16 +++++++++++----- scrapy/utils/signal.py | 10 ++++------ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3d838fb63..ee9d95836 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -268,6 +268,9 @@ spider_idle You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to prevent the spider from being closed. + Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider` + exception to provide a custom spider closing reason. + This signal does not support returning deferreds from its handlers. :param spider: the spider which has gone idle diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index dd3225082..949d89e76 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -15,7 +15,8 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper -from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning +from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning, \ + CloseSpider from scrapy.http import Response, Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider @@ -325,14 +326,19 @@ class ExecutionEngine: Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider exception, the spider is not closed until the next loop and this function is guaranteed to be called - (at least) once again. + (at least) once again. A handler can raise CloseSpider to provide a custom closing reason """ assert self.spider is not None # typing - res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider) - if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res): + expected_ex = (DontCloseSpider, CloseSpider) + res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex) + detected_ex = {ex: x.value + for _, x in res for ex in expected_ex + if isinstance(x, Failure) and isinstance(x.value, ex)} + if DontCloseSpider in detected_ex: return None if self.spider_is_idle(): - self.close_spider(self.spider, reason='finished') + reason = detected_ex[CloseSpider].reason if CloseSpider in detected_ex else 'finished' + self.close_spider(self.spider, reason=reason) def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: """Close (cancel) spider and clear all its outstanding requests""" diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 115707182..62808f3ce 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,5 +1,5 @@ """Helper functions for working with signals""" - +import collections import logging from twisted.internet.defer import DeferredList, Deferred @@ -16,15 +16,13 @@ from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) -class _IgnoredException(Exception): - pass - - def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. """ - dont_log = (named.pop('dont_log', _IgnoredException), StopDownload) + dont_log = named.pop('dont_log', ()) + dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,) + dont_log += (StopDownload, ) spider = named.get('spider', None) responses = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): From ce445f20462463e49bd9dfd417aa7397d4c17f1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Thu, 24 Jun 2021 09:56:05 +0100 Subject: [PATCH 551/568] Fix typing --- scrapy/core/engine.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 949d89e76..81a7a5068 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -337,8 +337,9 @@ class ExecutionEngine: if DontCloseSpider in detected_ex: return None if self.spider_is_idle(): - reason = detected_ex[CloseSpider].reason if CloseSpider in detected_ex else 'finished' - self.close_spider(self.spider, reason=reason) + ex = detected_ex.get(CloseSpider, CloseSpider(reason='finished')) + assert isinstance(ex, CloseSpider) # typing + self.close_spider(self.spider, reason=ex.reason) def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: """Close (cancel) spider and clear all its outstanding requests""" From 73ff9ffd64742bb4d64db376851b52cfc3658fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 26 Jun 2021 08:58:29 +0200 Subject: [PATCH 552/568] spiders.rst: indent warnings into class descriptions --- docs/topics/spiders.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index a3e9f410f..903fbd383 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -422,10 +422,9 @@ Crawling rules It receives a :class:`Twisted Failure ` instance as first parameter. - -.. warning:: Because of its internal implementation, you must explicitly set - callbacks for new requests when writing :class:`CrawlSpider`-based spiders; - unexpected behaviour can occur otherwise. + .. warning:: Because of its internal implementation, you must explicitly set + callbacks for new requests when writing :class:`CrawlSpider`-based spiders; + unexpected behaviour can occur otherwise. .. versionadded:: 2.0 The *errback* parameter. @@ -557,10 +556,9 @@ XMLFeedSpider item IDs. It receives a list of results and the response which originated those results. It must return a list of results (items or requests). - -.. warning:: Because of its internal implementation, you must explicitly set - callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders; - unexpected behaviour can occur otherwise. + .. warning:: Because of its internal implementation, you must explicitly set + callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders; + unexpected behaviour can occur otherwise. XMLFeedSpider example From f35970778b93033ee36379e0b8b55035aacaf918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Tue, 29 Jun 2021 13:21:38 +0100 Subject: [PATCH 553/568] Test case for raising CloseSpider on spider idle signal handler --- tests/test_engine.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index c200ded90..dc24f50fa 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -26,7 +26,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import ScrapyDeprecationWarning, CloseSpider from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor @@ -113,6 +113,18 @@ class ItemZeroDivisionErrorSpider(TestSpider): } +class ChangeCloseReasonSpider(TestSpider): + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(*args, **kwargs) + spider._set_crawler(crawler) + crawler.signals.connect(spider.spider_idle, signals.spider_idle) + return spider + + def spider_idle(self): + raise CloseSpider(reason="custom_reason") + + def start_test_site(debug=False): root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) @@ -251,6 +263,13 @@ class EngineTest(unittest.TestCase): yield self.run.run() self._assert_items_error() + @defer.inlineCallbacks + def test_crawler_change_close_reason_on_idle(self): + self.run = CrawlerRun(ChangeCloseReasonSpider) + yield self.run.run() + self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'}, + self.run.signals_caught[signals.spider_closed]) + def _assert_visited_urls(self): must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"] From e94d3ac173d2fcd269bdd8f06b6b1633953a225e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Tue, 29 Jun 2021 13:40:43 +0100 Subject: [PATCH 554/568] Expanded doc for idle signal --- docs/topics/signals.rst | 6 +++++- scrapy/core/engine.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index ee9d95836..530af1e37 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -269,7 +269,11 @@ spider_idle prevent the spider from being closed. Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider` - exception to provide a custom spider closing reason. + exception to provide a custom spider closing reason. An + idle handler is the perfect place to put some code that assesses + the final spider results and update the final closing reason + accordingly (e.g. setting it to 'too_few_results' instead of + 'finished'). This signal does not support returning deferreds from its handlers. diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 81a7a5068..0b34d213d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -326,7 +326,7 @@ class ExecutionEngine: Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider exception, the spider is not closed until the next loop and this function is guaranteed to be called - (at least) once again. A handler can raise CloseSpider to provide a custom closing reason + (at least) once again. A handler can raise CloseSpider to provide a custom closing reason. """ assert self.spider is not None # typing expected_ex = (DontCloseSpider, CloseSpider) From 7597d860c86e0ddfaf8e3ce57492106c907245f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Thu, 8 Jul 2021 12:39:17 +0100 Subject: [PATCH 555/568] Update scrapy/core/engine.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/core/engine.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 0b34d213d..b7f91d744 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -15,8 +15,11 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper -from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning, \ - CloseSpider +from scrapy.exceptions import ( + CloseSpider, + DontCloseSpider, + ScrapyDeprecationWarning, +) from scrapy.http import Response, Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider From 6b8f694653b0eec95055d50fafde0ef2e0ec8bc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Thu, 8 Jul 2021 12:40:02 +0100 Subject: [PATCH 556/568] Update scrapy/core/engine.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/core/engine.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b7f91d744..b04fedce6 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -334,9 +334,12 @@ class ExecutionEngine: assert self.spider is not None # typing expected_ex = (DontCloseSpider, CloseSpider) res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex) - detected_ex = {ex: x.value - for _, x in res for ex in expected_ex - if isinstance(x, Failure) and isinstance(x.value, ex)} + detected_ex = { + ex: x.value + for _, x in res + for ex in expected_ex + if isinstance(x, Failure) and isinstance(x.value, ex) + } if DontCloseSpider in detected_ex: return None if self.spider_is_idle(): From eca641aa3da335781c373e3d50deb78837044c16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Thu, 8 Jul 2021 12:40:20 +0100 Subject: [PATCH 557/568] Update tests/test_engine.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- tests/test_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index dc24f50fa..92bf45f25 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -26,7 +26,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import ScrapyDeprecationWarning, CloseSpider +from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor From cb08e3644b081febecd90eec9391367a2d13ee5b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 8 Jul 2021 09:22:21 -0300 Subject: [PATCH 558/568] Remove trailing whitespaces --- scrapy/core/engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b04fedce6..f9de7ee23 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -17,9 +17,9 @@ from scrapy import signals from scrapy.core.scraper import Scraper from scrapy.exceptions import ( CloseSpider, - DontCloseSpider, + DontCloseSpider, ScrapyDeprecationWarning, -) +) from scrapy.http import Response, Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider From c062ed017a89e40c41140ff2782043776b26165b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 12 Jul 2021 13:34:22 -0300 Subject: [PATCH 559/568] [CI] fail-fast: false (#5200) --- .github/workflows/checks.yml | 1 + .github/workflows/tests-macos.yml | 1 + .github/workflows/tests-ubuntu.yml | 1 + .github/workflows/tests-windows.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 02c647da9..e7080db9a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -5,6 +5,7 @@ jobs: checks: runs-on: ubuntu-18.04 strategy: + fail-fast: false matrix: include: - python-version: 3.9 diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 4f8f7a19d..095ca1013 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -5,6 +5,7 @@ jobs: tests: runs-on: macos-10.15 strategy: + fail-fast: false matrix: python-version: [3.6, 3.7, 3.8, 3.9] diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index df5ee9d69..b42e8b127 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -5,6 +5,7 @@ jobs: tests: runs-on: ubuntu-18.04 strategy: + fail-fast: false matrix: include: - python-version: 3.7 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 5459a845b..30fda33e8 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -5,6 +5,7 @@ jobs: tests: runs-on: windows-latest strategy: + fail-fast: false matrix: include: - python-version: 3.6 From 4ddc9d6b55fa708becfd70812ea44eb0c6837638 Mon Sep 17 00:00:00 2001 From: D R Siddhartha Date: Tue, 13 Jul 2021 20:52:29 +0530 Subject: [PATCH 560/568] Feeds: Item Filters (#5178) --- docs/topics/feed-exports.rst | 54 +++++++++++++++ scrapy/extensions/feedexport.py | 48 ++++++++++++- tests/test_feedexport.py | 118 ++++++++++++++++++++++++++++++-- 3 files changed, 214 insertions(+), 6 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 26c247cdd..7a4b054e9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -268,6 +268,45 @@ in multiple files, with the specified maximum item count per file. That way, as soon as a file reaches the maximum item count, that file is delivered to the feed URI, allowing item delivery to start way before the end of the crawl. +.. _item-filter: + +Item filtering +============== + +.. versionadded:: VERSION + +You can filter items that you want to allow for a particular feed by using the +``item_classes`` option in :ref:`feeds options `. Only items of +the specified types will be added to the feed. + +The ``item_classes`` option is implemented by the :class:`~scrapy.extensions.feedexport.ItemFilter` +class, which is the default value of the ``item_filter`` :ref:`feed option `. + +You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s +method ``accepts`` and taking ``feed_options`` as an argument. + +For instance:: + + class MyCustomFilter: + + def __init__(self, feed_options): + self.feed_options = feed_options + + def accepts(self, item): + if "field1" in item and item["field1"] == "expected_data": + return True + return False + + +You can assign your custom filtering class to the ``item_filter`` :ref:`option of a feed `. +See :setting:`FEEDS` for examples. + +ItemFilter +---------- + +.. autoclass:: scrapy.extensions.feedexport.ItemFilter + :members: + Settings ======== @@ -311,6 +350,7 @@ For instance:: 'format': 'json', 'encoding': 'utf8', 'store_empty': False, + 'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'], 'fields': None, 'indent': 4, 'item_export_kwargs': { @@ -320,12 +360,14 @@ For instance:: '/home/user/documents/items.xml': { 'format': 'xml', 'fields': ['name', 'price'], + 'item_filter': MyCustomFilter1, 'encoding': 'latin1', 'indent': 8, }, pathlib.Path('items.csv'): { 'format': 'csv', 'fields': ['price', 'name'], + 'item_filter': 'myproject.filters.MyCustomFilter2', }, } @@ -347,6 +389,18 @@ as a fallback value if that key is not provided for a specific feed definition: - ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. +- ``item_classes``: list of :ref:`item classes ` to export. + + If undefined or empty, all items are exported. + + .. versionadded:: VERSION + +- ``item_filter``: a :ref:`filter class ` to filter items to export. + + :class:`~scrapy.extensions.feedexport.ItemFilter` is used be default. + + .. versionadded:: VERSION + - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. - ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index bec114707..89dca12f4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -46,6 +46,38 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): return builder(*preargs, uri, *args, **kwargs) +class ItemFilter: + """ + This will be used by FeedExporter to decide if an item should be allowed + to be exported to a particular feed. + + :param feed_options: feed specific options passed from FeedExporter + :type feed_options: dict + """ + + def __init__(self, feed_options): + self.feed_options = feed_options + self.item_classes = set() + + if 'item_classes' in self.feed_options: + for item_class in self.feed_options['item_classes']: + self.item_classes.add(load_object(item_class)) + + def accepts(self, item): + """ + Return ``True`` if `item` should be exported or ``False`` otherwise. + + :param item: scraped item which user wants to check if is acceptable + :type item: :ref:`Scrapy items ` + :return: `True` if accepted, `False` otherwise + :rtype: bool + """ + if self.item_classes: + return isinstance(item, tuple(self.item_classes)) + + return True # accept all items if none declared in item_classes + + class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" @@ -215,7 +247,7 @@ class FTPFeedStorage(BlockingFeedStorage): class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template): + def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template, filter): self.file = file self.exporter = exporter self.storage = storage @@ -225,6 +257,7 @@ class _FeedSlot: self.store_empty = store_empty self.uri_template = uri_template self.uri = uri + self.filter = filter # flags self.itemcount = 0 self._exporting = False @@ -255,6 +288,7 @@ class FeedExporter: self.settings = crawler.settings self.feeds = {} self.slots = [] + self.filters = {} if not self.settings['FEEDS'] and not self.settings['FEED_URI']: raise NotConfigured @@ -269,12 +303,14 @@ class FeedExporter: uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) + self.filters[uri] = self._load_filter(feed_options) # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' for uri, feed_options in self.settings.getdict('FEEDS').items(): uri = str(uri) # handle pathlib.Path objects self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) + self.filters[uri] = self._load_filter(feed_options) self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') @@ -368,6 +404,7 @@ class FeedExporter: store_empty=feed_options['store_empty'], batch_id=batch_id, uri_template=uri_template, + filter=self.filters[uri_template] ) if slot.store_empty: slot.start_exporting() @@ -376,6 +413,10 @@ class FeedExporter: def item_scraped(self, item, spider): slots = [] for slot in self.slots: + if not slot.filter.accepts(item): + slots.append(slot) # if slot doesn't accept item, continue with next slot + continue + slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 @@ -486,3 +527,8 @@ class FeedExporter: uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) return params + + def _load_filter(self, feed_options): + # load the item filter if declared else load the default filter class + item_filter_class = load_object(feed_options.get("item_filter", ItemFilter)) + return item_filter_class(feed_options) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index df7ec4461..81437b011 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -561,6 +561,10 @@ class FeedExportTestBase(ABC, unittest.TestCase): egg = scrapy.Field() baz = scrapy.Field() + class MyItem2(scrapy.Item): + foo = scrapy.Field() + hello = scrapy.Field() + def _random_temp_filename(self, inter_dir=''): chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) @@ -888,13 +892,9 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_item_classes(self): - class MyItem2(scrapy.Item): - foo = scrapy.Field() - hello = scrapy.Field() - items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), - MyItem2({'hello': 'world2', 'foo': 'bar2'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}), {'hello': 'world4', 'egg': 'spam4'}, ] @@ -929,6 +929,114 @@ class FeedExportTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings, ordered=True) + @defer.inlineCallbacks + def test_export_based_on_item_classes(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), + {'hello': 'world3', 'egg': 'spam3'}, + ] + + formats = { + 'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n', + 'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]', + 'jsonlines': ( + b'{"foo": "bar1", "egg": "spam1"}\n' + b'{"hello": "world2", "foo": "bar2"}\n' + ), + 'xml': ( + b'\n\n' + b'bar1spam1\n' + b'world2bar2\nworld3' + b'spam3\n' + ), + } + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'csv', + 'item_classes': [self.MyItem], + }, + self._random_temp_filename(): { + 'format': 'json', + 'item_classes': [self.MyItem2], + }, + self._random_temp_filename(): { + 'format': 'jsonlines', + 'item_classes': [self.MyItem, self.MyItem2], + }, + self._random_temp_filename(): { + 'format': 'xml', + }, + }, + } + + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + self.assertEqual(expected, data[fmt]) + + @defer.inlineCallbacks + def test_export_based_on_custom_filters(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), + {'hello': 'world3', 'egg': 'spam3'}, + ] + + MyItem = self.MyItem + + class CustomFilter1: + def __init__(self, feed_options): + pass + + def accepts(self, item): + return isinstance(item, MyItem) + + class CustomFilter2(scrapy.extensions.feedexport.ItemFilter): + def accepts(self, item): + if 'foo' not in item.fields: + return False + return True + + class CustomFilter3(scrapy.extensions.feedexport.ItemFilter): + def accepts(self, item): + if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1": + return True + return False + + formats = { + 'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]', + 'xml': ( + b'\n\n' + b'bar1spam1\n' + b'world2bar2\n' + ), + 'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n', + } + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'json', + 'item_filter': CustomFilter1, + }, + self._random_temp_filename(): { + 'format': 'xml', + 'item_filter': CustomFilter2, + }, + self._random_temp_filename(): { + 'format': 'jsonlines', + 'item_classes': [self.MyItem, self.MyItem2], + 'item_filter': CustomFilter3, + }, + }, + } + + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + self.assertEqual(expected, data[fmt]) + @defer.inlineCallbacks def test_export_dicts(self): # When dicts are used, only keys from the first row are used as From fcc6becc586e6f895cdeed66a579d5735d215de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=BCrkalp=20Burak=20KAYRANCIO=C4=9ELU?= Date: Wed, 14 Jul 2021 11:00:43 +0300 Subject: [PATCH 561/568] S3FeedStorage: allow custom endpoint (#4998) Co-authored-by: Andrey Rahmatullin --- docs/topics/feed-exports.rst | 3 ++- scrapy/extensions/feedexport.py | 7 ++++-- tests/test_feedexport.py | 44 +++++++++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7a4b054e9..af7fce852 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -201,9 +201,10 @@ passed through the following settings: - :setting:`AWS_ACCESS_KEY_ID` - :setting:`AWS_SECRET_ACCESS_KEY` -You can also define a custom ACL for exported feeds using this setting: +You can also define a custom ACL and custom endpoint for exported feeds using this setting: - :setting:`FEED_STORAGE_S3_ACL` +- :setting:`AWS_ENDPOINT_URL` This storage backend uses :ref:`delayed file delivery `. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 89dca12f4..84a79e32d 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -150,7 +150,7 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None, acl=None, *, + def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *, feed_options=None): if not is_botocore_available(): raise NotConfigured('missing botocore library') @@ -160,11 +160,13 @@ class S3FeedStorage(BlockingFeedStorage): self.secret_key = u.password or secret_key self.keyname = u.path[1:] # remove first "/" self.acl = acl + self.endpoint_url = endpoint_url import botocore.session session = botocore.session.get_session() self.s3_client = session.create_client( 's3', aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key) + aws_secret_access_key=self.secret_key, + endpoint_url=self.endpoint_url) if feed_options and feed_options.get('overwrite', True) is False: logger.warning('S3 does not support appending to files. To ' 'suppress this warning, remove the overwrite ' @@ -178,6 +180,7 @@ class S3FeedStorage(BlockingFeedStorage): access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, + endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None, feed_options=feed_options, ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 81437b011..da0b2c786 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -326,6 +326,17 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + def test_init_with_endpoint_url(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + endpoint_url='https://example.com' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, 'https://example.com') + def test_from_crawler_without_acl(self): settings = { 'AWS_ACCESS_KEY_ID': 'access_key', @@ -340,6 +351,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) + def test_without_endpoint_url(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, None) + def test_from_crawler_with_acl(self): settings = { 'AWS_ACCESS_KEY_ID': 'access_key', @@ -355,6 +380,21 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + def test_from_crawler_with_endpoint_url(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'AWS_ENDPOINT_URL': 'https://example.com', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, 'https://example.com') + @defer.inlineCallbacks def test_store_botocore_without_acl(self): skip_if_no_boto() @@ -1917,8 +1957,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): class S3FeedStorageWithoutFeedOptions(S3FeedStorage): - def __init__(self, uri, access_key, secret_key, acl): - super().__init__(uri, access_key, secret_key, acl) + def __init__(self, uri, access_key, secret_key, acl, endpoint_url): + super().__init__(uri, access_key, secret_key, acl, endpoint_url) class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): From d7deba7e89242774ae712d87eb7bb331759a731f Mon Sep 17 00:00:00 2001 From: Marlena Chatzigrigoriou <56519084+marlenachatzigrigoriou@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:34:28 +0300 Subject: [PATCH 562/568] Document all import paths and use the shortest in examples (#5099) --- docs/faq.rst | 4 +- docs/intro/tutorial.rst | 32 +++++++-------- docs/topics/api.rst | 4 +- docs/topics/contracts.rst | 4 +- docs/topics/coroutines.rst | 2 +- docs/topics/debug.rst | 2 +- docs/topics/developer-tools.rst | 2 +- docs/topics/downloader-middleware.rst | 58 +++++++++++++-------------- docs/topics/dynamic-content.rst | 14 +++---- docs/topics/exporters.rst | 4 +- docs/topics/feed-exports.rst | 4 +- docs/topics/item-pipeline.rst | 6 +-- docs/topics/items.rst | 10 +++-- docs/topics/jobs.rst | 4 +- docs/topics/leaks.rst | 16 ++++---- docs/topics/logging.rst | 2 +- docs/topics/media-pipeline.rst | 6 +-- docs/topics/request-response.rst | 16 ++++---- docs/topics/selectors.rst | 8 ++-- docs/topics/settings.rst | 16 ++++---- docs/topics/shell.rst | 8 ++-- docs/topics/signals.rst | 54 ++++++++++++------------- docs/topics/spider-middleware.rst | 26 ++++++------ docs/topics/spiders.rst | 43 ++++++++++---------- 24 files changed, 175 insertions(+), 170 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 9709885f6..16903daea 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,7 +149,7 @@ How can I prevent memory errors due to many allowed domains? ------------------------------------------------------------ If you have a spider with a long list of -:attr:`~scrapy.spiders.Spider.allowed_domains` (e.g. 50,000+), consider +:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider replacing the default :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware with a :ref:`custom spider middleware ` that requires @@ -157,7 +157,7 @@ less memory. For example: - If your domain names are similar enough, use your own regular expression instead joining the strings in - :attr:`~scrapy.spiders.Spider.allowed_domains` into a complex regular + :attr:`~scrapy.Spider.allowed_domains` into a complex regular expression. - If you can `meet the installation requirements`_, use pyre2_ instead of diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 740e47d0c..438f3d6df 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -78,7 +78,7 @@ Our first Spider Spiders are classes that you define and that Scrapy uses to scrape information from a website (or a group of websites). They must subclass -:class:`~scrapy.spiders.Spider` and define the initial requests to make, +:class:`~scrapy.Spider` and define the initial requests to make, optionally how to follow links in the pages, and how to parse the downloaded page content to extract data. @@ -107,26 +107,26 @@ This is the code for our first Spider. Save it in a file named self.log(f'Saved file {filename}') -As you can see, our Spider subclasses :class:`scrapy.Spider ` +As you can see, our Spider subclasses :class:`scrapy.Spider ` and defines some attributes and methods: -* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be +* :attr:`~scrapy.Spider.name`: identifies the Spider. It must be unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of +* :meth:`~scrapy.Spider.start_requests`: must return an iterable of Requests (you can return a list of requests or write a generator function) which the Spider will begin to crawl from. Subsequent requests will be generated successively from these initial requests. -* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle +* :meth:`~scrapy.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter is an instance of :class:`~scrapy.http.TextResponse` that holds the page content and has further helpful methods to handle it. - The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting + The :meth:`~scrapy.Spider.parse` method usually parses the response, extracting the scraped data as dicts and also finding new URLs to - follow and creating new requests (:class:`~scrapy.http.Request`) from them. + follow and creating new requests (:class:`~scrapy.Request`) from them. How to run our spider --------------------- @@ -162,7 +162,7 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy schedules the :class:`scrapy.Request ` objects +Scrapy schedules the :class:`scrapy.Request ` objects returned by the ``start_requests`` method of the Spider. Upon receiving a response for each one, it instantiates :class:`~scrapy.http.Response` objects and calls the callback method associated with the request (in this case, the @@ -171,11 +171,11 @@ and calls the callback method associated with the request (in this case, the A shortcut to the start_requests method --------------------------------------- -Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method -that generates :class:`scrapy.Request ` objects from URLs, -you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute +Instead of implementing a :meth:`~scrapy.Spider.start_requests` method +that generates :class:`scrapy.Request ` objects from URLs, +you can just define a :attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This list will then be used by the default implementation -of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests +of :meth:`~scrapy.Spider.start_requests` to create the initial requests for your spider:: import scrapy @@ -194,9 +194,9 @@ for your spider:: with open(filename, 'wb') as f: f.write(response.body) -The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each +The :meth:`~scrapy.Spider.parse` method will be called to handle each of the requests for those URLs, even though we haven't explicitly told Scrapy -to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's +to do so. This happens because :meth:`~scrapy.Spider.parse` is Scrapy's default callback method, which is called for requests without an explicitly assigned callback. @@ -248,7 +248,7 @@ object: The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of -:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements +:class:`~scrapy.Selector` objects that wrap around XML/HTML elements and allow you to run further queries to fine-grain the selection or extract the data. @@ -670,7 +670,7 @@ the pagination links with the ``parse`` callback as we saw before. Here we're passing callbacks to :meth:`response.follow_all ` as positional arguments to make the code shorter; it also works for -:class:`~scrapy.http.Request`. +:class:`~scrapy.Request`. The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 445b2979f..900b19c7a 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -29,7 +29,7 @@ how you :ref:`configure the downloader middlewares .. class:: Crawler(spidercls, settings) The Crawler object must be instantiated with a - :class:`scrapy.spiders.Spider` subclass and a + :class:`scrapy.Spider` subclass and a :class:`scrapy.settings.Settings` object. .. attribute:: settings @@ -196,7 +196,7 @@ SpiderLoader API match the request's url against the domains of the spiders. :param request: queried request - :type request: :class:`~scrapy.http.Request` instance + :type request: :class:`~scrapy.Request` instance .. _topics-api-signals: diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index e61421bf1..ef296dc9e 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -37,7 +37,7 @@ This callback is tested using three built-in contracts: .. class:: CallbackKeywordArgumentsContract - This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` + This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` attribute for the sample request. It must be a valid JSON dictionary. :: @@ -88,7 +88,7 @@ override three methods: .. method:: Contract.adjust_request_args(args) This receives a ``dict`` as an argument containing default arguments - for request object. :class:`~scrapy.http.Request` is used by default, + for request object. :class:`~scrapy.Request` is used by default, but this can be changed with the ``request_cls`` attribute. If multiple contracts in chain have this attribute defined, the last one is used. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 3b1549bd3..0904637b0 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -15,7 +15,7 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): -- :class:`~scrapy.http.Request` callbacks. +- :class:`~scrapy.Request` callbacks. .. note:: The callback output is not processed until the whole callback finishes. diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index d75f17301..4d452b4df 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -36,7 +36,7 @@ Consider the following Scrapy spider below:: Basically this is a simple spider which parses two pages of items (the start_urls). Items also have a details page with additional information, so we -use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a +use the ``cb_kwargs`` functionality of :class:`~scrapy.Request` to pass a partially populated item. diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index c83b1a9d9..057b1ec62 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -274,7 +274,7 @@ In more complex websites, it could be difficult to easily reproduce the requests, as we could need to add ``headers`` or ``cookies`` to make it work. In those cases you can export the requests in `cURL `_ format, by right-clicking on each of them in the network tool and using the -:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent +:meth:`~scrapy.Request.from_curl()` method to generate an equivalent request:: from scrapy import Request diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index b539c23df..80c6c2c37 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -76,7 +76,7 @@ object gives you access, for example, to the :ref:`settings `. middleware. :meth:`process_request` should either: return ``None``, return a - :class:`~scrapy.http.Response` object, return a :class:`~scrapy.http.Request` + :class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request` object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`. If it returns ``None``, Scrapy will continue processing this request, executing all @@ -88,7 +88,7 @@ object gives you access, for example, to the :ref:`settings `. or the appropriate download function; it'll return that response. The :meth:`process_response` methods of installed middleware is always called on every response. - If it returns a :class:`~scrapy.http.Request` object, Scrapy will stop calling + If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling process_request methods and reschedule the returned request. Once the newly returned request is performed, the appropriate middleware chain will be called on the downloaded response. @@ -100,22 +100,22 @@ object gives you access, for example, to the :ref:`settings `. ignored and not logged (unlike other exceptions). :param request: the request being processed - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_response(request, response, spider) :meth:`process_response` should either: return a :class:`~scrapy.http.Response` - object, return a :class:`~scrapy.http.Request` object or + object, return a :class:`~scrapy.Request` object or raise a :exc:`~scrapy.exceptions.IgnoreRequest` exception. If it returns a :class:`~scrapy.http.Response` (it could be the same given response, or a brand-new one), that response will continue to be processed with the :meth:`process_response` of the next middleware in the chain. - If it returns a :class:`~scrapy.http.Request` object, the middleware chain is + If it returns a :class:`~scrapy.Request` object, the middleware chain is halted and the returned request is rescheduled to be downloaded in the future. This is the same behavior as if a request is returned from :meth:`process_request`. @@ -124,13 +124,13 @@ object gives you access, for example, to the :ref:`settings `. exception, it is ignored and not logged (unlike other exceptions). :param request: the request that originated the response - :type request: is a :class:`~scrapy.http.Request` object + :type request: is a :class:`~scrapy.Request` object :param response: the response being processed :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_exception(request, exception, spider) @@ -139,7 +139,7 @@ object gives you access, for example, to the :ref:`settings `. exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception) :meth:`process_exception` should return: either ``None``, - a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.http.Request` object. + a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object. If it returns ``None``, Scrapy will continue processing this exception, executing any other :meth:`process_exception` methods of installed middleware, @@ -149,19 +149,19 @@ object gives you access, for example, to the :ref:`settings `. method chain of installed middleware is started, and Scrapy won't bother calling any other :meth:`process_exception` methods of middleware. - If it returns a :class:`~scrapy.http.Request` object, the returned request is + If it returns a :class:`~scrapy.Request` object, the returned request is rescheduled to be downloaded in the future. This stops the execution of :meth:`process_exception` methods of the middleware the same as returning a response would. :param request: the request that generated the exception - :type request: is a :class:`~scrapy.http.Request` object + :type request: is a :class:`~scrapy.Request` object :param exception: the raised exception :type exception: an ``Exception`` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: from_crawler(cls, crawler) @@ -203,13 +203,13 @@ CookiesMiddleware browsers do. .. caution:: When non-UTF8 encoded byte sequences are passed to a - :class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log + :class:`~scrapy.Request`, the ``CookiesMiddleware`` will log a warning. Refer to :ref:`topics-logging-advanced-customization` to customize the logging behaviour. .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. The following settings can be used to configure the cookie middleware: @@ -258,7 +258,7 @@ web server and received cookies in :class:`~scrapy.http.Response` will **not** be merged with the existing cookies. For more detailed information see the ``cookies`` parameter in -:class:`~scrapy.http.Request`. +:class:`~scrapy.Request`. .. setting:: COOKIES_DEBUG @@ -501,7 +501,7 @@ defines the methods described below. the :signal:`open_spider ` signal. :param spider: the spider which has been opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: close_spider(spider) @@ -509,27 +509,27 @@ defines the methods described below. the :signal:`close_spider ` signal. :param spider: the spider which has been closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: retrieve_response(spider, request) Return response if present in cache, or ``None`` otherwise. :param spider: the spider which generated the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param request: the request to find cached response for - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object .. method:: store_response(spider, request, response) Store the given response in the cache. :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param request: the corresponding request the spider generated - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param response: the response to store in the cache :type response: :class:`~scrapy.http.Response` object @@ -722,7 +722,7 @@ HttpProxyMiddleware .. class:: HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the - ``proxy`` meta value for :class:`~scrapy.http.Request` objects. + ``proxy`` meta value for :class:`~scrapy.Request` objects. Like the Python standard library module :mod:`urllib.request`, it obeys the following environment variables: @@ -749,12 +749,12 @@ RedirectMiddleware .. reqmeta:: redirect_urls The urls which the request goes through (while being redirected) can be found -in the ``redirect_urls`` :attr:`Request.meta ` key. +in the ``redirect_urls`` :attr:`Request.meta ` key. .. reqmeta:: redirect_reasons The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the -``redirect_reasons`` :attr:`Request.meta ` key. For +``redirect_reasons`` :attr:`Request.meta ` key. For example: ``[301, 302, 307, 'meta refresh']``. The format of a reason depends on the middleware that handled the corresponding @@ -770,7 +770,7 @@ settings (see the settings documentation for more info): .. reqmeta:: dont_redirect -If :attr:`Request.meta ` has ``dont_redirect`` +If :attr:`Request.meta ` has ``dont_redirect`` key set to True, the request will be ignored by this middleware. If you want to handle some redirect status codes in your spider, you can @@ -783,7 +783,7 @@ responses (and pass them through to your spider) you can do this:: handle_httpstatus_list = [301, 302] The ``handle_httpstatus_list`` key of :attr:`Request.meta -` can also be used to specify which response codes to +` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` to ``True`` if you want to allow any response code for a request. @@ -889,7 +889,7 @@ settings (see the settings documentation for more info): .. reqmeta:: dont_retry -If :attr:`Request.meta ` has ``dont_retry`` key +If :attr:`Request.meta ` has ``dont_retry`` key set to True, the request will be ignored by this middleware. To retry requests from a spider callback, you can use the @@ -919,7 +919,7 @@ Default: ``2`` Maximum number of times to retry, in addition to the first download. Maximum number of retries can also be specified per-request using -:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. +:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. When initialized, the :reqmeta:`max_retry_times` meta key takes higher precedence over the :setting:`RETRY_TIMES` setting. @@ -986,7 +986,7 @@ RobotsTxtMiddleware .. reqmeta:: dont_obey_robotstxt -If :attr:`Request.meta ` has +If :attr:`Request.meta ` has ``dont_obey_robotstxt`` key set to True the request will be ignored by this middleware even if :setting:`ROBOTSTXT_OBEY` is enabled. diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 495111b56..aa32c8943 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -62,9 +62,9 @@ download the webpage with an HTTP client like curl_ or wget_ and see if the information can be found in the response they get. If they get a response with the desired data, modify your Scrapy -:class:`~scrapy.http.Request` to match that of the other HTTP client. For +:class:`~scrapy.Request` to match that of the other HTTP client. For example, try using the same user-agent string (:setting:`USER_AGENT`) or the -same :attr:`~scrapy.http.Request.headers`. +same :attr:`~scrapy.Request.headers`. If they also get a response without the desired data, you’ll need to take steps to make your request more similar to that of the web browser. See @@ -81,14 +81,14 @@ Use the :ref:`network tool ` of your web browser to see how your web browser performs the desired request, and try to reproduce that request with Scrapy. -It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP +It might be enough to yield a :class:`~scrapy.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and -form parameters (see :class:`~scrapy.http.FormRequest`) of that request. +form parameters (see :class:`~scrapy.FormRequest`) of that request. As all major browsers allow to export the requests in `cURL `_ format, Scrapy incorporates the method -:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent -:class:`~scrapy.http.Request` from a cURL command. To get more information +:meth:`~scrapy.Request.from_curl()` to generate an equivalent +:class:`~scrapy.Request` from a cURL command. To get more information visit :ref:`request from curl ` inside the network tool section. @@ -125,7 +125,7 @@ data from it depends on the type of response: If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a - :class:`~scrapy.selector.Selector` and then + :class:`~scrapy.Selector` and then :ref:`use it ` as usual:: selector = Selector(data['html']) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 8648daded..8c30122b6 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -90,7 +90,7 @@ described next. 1. Declaring a serializer in the field -------------------------------------- -If you use :class:`~.Item` you can declare a serializer in the +If you use :class:`~scrapy.Item` you can declare a serializer in the :ref:`field metadata `. The serializer must be a callable which receives a value and returns its serialized form. @@ -172,7 +172,7 @@ BaseItemExporter :param field: the field being serialized. If the source :ref:`item object ` does not define field metadata, *field* is an empty :class:`dict`. - :type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance + :type field: :class:`~scrapy.Field` object or a :class:`dict` instance :param name: the name of the field being serialized :type name: str diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index af7fce852..216a8bc52 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -674,9 +674,9 @@ The function signature should be as follows: :type params: dict :param spider: source spider of the feed items - :type spider: scrapy.spiders.Spider + :type spider: scrapy.Spider -For example, to include the :attr:`name ` of the +For example, to include the :attr:`name ` of the source spider in the feed URI: #. Define the following function somewhere in your project:: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 6287ee0ad..5351a2293 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -42,7 +42,7 @@ Each item pipeline component is a Python class that must implement the following :type item: :ref:`item object ` :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object Additionally, they may also implement the following methods: @@ -51,14 +51,14 @@ Additionally, they may also implement the following methods: This method is called when the spider is opened. :param spider: the spider which was opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: close_spider(self, spider) This method is called when the spider is closed. :param spider: the spider which was closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: from_crawler(cls, crawler) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 65bf156ac..7cd482d07 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -42,7 +42,8 @@ Item objects :class:`Item` provides a :class:`dict`-like API plus additional features that make it the most feature-complete item type: -.. class:: Item([arg]) +.. class:: scrapy.item.Item([arg]) +.. class:: scrapy.Item([arg]) :class:`Item` objects replicate the standard :class:`dict` API, including its ``__init__`` method. @@ -199,7 +200,8 @@ It's important to note that the :class:`Field` objects used to declare the item do not stay assigned as class attributes. Instead, they can be accessed through the :attr:`Item.fields` attribute. -.. class:: Field([arg]) +.. class:: scrapy.item.Field([arg]) +.. class:: scrapy.Field([arg]) The :class:`Field` class is just an alias to the built-in :class:`dict` class and doesn't provide any extra functionality or attributes. In other words, @@ -317,11 +319,11 @@ If that is not the desired behavior, use a deep copy instead. See :mod:`copy` for more information. To create a shallow copy of an item, you can either call -:meth:`~scrapy.item.Item.copy` on an existing item +:meth:`~scrapy.Item.copy` on an existing item (``product2 = product.copy()``) or instantiate your item class from an existing item (``product2 = Product(product)``). -To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead +To create a deep copy, call :meth:`~scrapy.Item.deepcopy` instead (``product2 = product.deepcopy()``). diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index d855d0133..e49f37a2f 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -74,10 +74,10 @@ on cookies. Request serialization --------------------- -For persistence to work, :class:`~scrapy.http.Request` objects must be +For persistence to work, :class:`~scrapy.Request` objects must be serializable with :mod:`pickle`, except for the ``callback`` and ``errback`` values passed to their ``__init__`` method, which must be methods of the -running :class:`~scrapy.spiders.Spider` class. +running :class:`~scrapy.Spider` class. If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index b895b95cb..477652704 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -27,7 +27,7 @@ Common causes of memory leaks It happens quite often (sometimes by accident, sometimes on purpose) that the Scrapy developer passes objects referenced in Requests (for example, using the -:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta` +:attr:`~scrapy.Request.cb_kwargs` or :attr:`~scrapy.Request.meta` attributes or the request callback function) and that effectively bounds the lifetime of those referenced objects to the lifetime of the Request. This is, by far, the most common cause of memory leaks in Scrapy projects, and a quite @@ -48,9 +48,9 @@ Too Many Requests? ------------------ By default Scrapy keeps the request queue in memory; it includes -:class:`~scrapy.http.Request` objects and all objects -referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs` -and :attr:`~scrapy.http.Request.meta`). +:class:`~scrapy.Request` objects and all objects +referenced in Request attributes (e.g. in :attr:`~scrapy.Request.cb_kwargs` +and :attr:`~scrapy.Request.meta`). While not necessarily a leak, this can take a lot of memory. Enabling :ref:`persistent job queue ` could help keeping memory usage in control. @@ -90,11 +90,11 @@ Which objects are tracked? The objects tracked by ``trackrefs`` are all from these classes (and all its subclasses): -* :class:`scrapy.http.Request` +* :class:`scrapy.Request` * :class:`scrapy.http.Response` -* :class:`scrapy.item.Item` -* :class:`scrapy.selector.Selector` -* :class:`scrapy.spiders.Spider` +* :class:`scrapy.Item` +* :class:`scrapy.Selector` +* :class:`scrapy.Spider` A real example -------------- diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 00806392a..dda04dc4d 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -93,7 +93,7 @@ path:: Logging from Spiders ==================== -Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider +Scrapy provides a :data:`~scrapy.Spider.logger` within each Spider instance, which can be accessed and used like this:: import scrapy diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 156897274..3438cb637 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -259,7 +259,7 @@ respectively), the pipeline will put the results under the respective field When using :ref:`item types ` for which fields are defined beforehand, you must define both the URLs field and the results field. For example, when using the images pipeline, items must define both the ``image_urls`` and the -``images`` field. For instance, using the :class:`~scrapy.item.Item` class:: +``images`` field. For instance, using the :class:`~scrapy.Item` class:: import scrapy @@ -424,7 +424,7 @@ See here the methods that you can override in your custom Files Pipeline: In addition to ``response``, this method receives the original :class:`request `, :class:`info ` and - :class:`item ` + :class:`item ` You can override this method to customize the download path of each file. @@ -563,7 +563,7 @@ See here the methods that you can override in your custom Images Pipeline: In addition to ``response``, this method receives the original :class:`request `, :class:`info ` and - :class:`item ` + :class:`item ` You can override this method to customize the download path of each file. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 73b5a858f..a6a3daf31 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -35,7 +35,7 @@ Request objects request (once it's downloaded) as its first parameter. For more information see :ref:`topics-request-response-ref-request-callback-arguments` below. If a Request doesn't specify a callback, the spider's - :meth:`~scrapy.spiders.Spider.parse` method will be used. + :meth:`~scrapy.Spider.parse` method will be used. Note that if exceptions are raised during processing, errback is called instead. :type callback: collections.abc.Callable @@ -60,7 +60,7 @@ Request objects .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. :type headers: dict @@ -92,7 +92,7 @@ Request objects To create a request that does not send stored cookies and does not store received cookies, set the ``dont_merge_cookies`` key to ``True`` - in :attr:`request.meta `. + in :attr:`request.meta `. Example of a request that sends manually-defined cookies and ignores cookie storage:: @@ -107,7 +107,7 @@ Request objects .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. :type cookies: dict or list @@ -495,7 +495,9 @@ fields with form data from :class:`Response` objects. .. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms -.. class:: FormRequest(url, [formdata, ...]) +.. class:: scrapy.http.request.form.FormRequest +.. class:: scrapy.http.FormRequest +.. class:: scrapy.FormRequest(url, [formdata, ...]) The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The remaining arguments are the same as for the :class:`Request` class and are @@ -694,7 +696,7 @@ Response objects :param request: the initial value of the :attr:`Response.request` attribute. This represents the :class:`Request` that generated this response. - :type request: scrapy.http.Request + :type request: scrapy.Request :param certificate: an object representing the server's SSL certificate. :type certificate: twisted.internet.ssl.Certificate @@ -920,7 +922,7 @@ TextResponse objects .. attribute:: TextResponse.selector - A :class:`~scrapy.selector.Selector` instance using the response as + A :class:`~scrapy.Selector` instance using the response as target. The selector is lazily instantiated on first access. :class:`TextResponse` objects support the following methods in addition to diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 9caba5ee5..574d4568c 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -48,7 +48,7 @@ Constructing selectors .. highlight:: python -Response objects expose a :class:`~scrapy.selector.Selector` instance +Response objects expose a :class:`~scrapy.Selector` instance on ``.selector`` attribute: >>> response.selector.xpath('//span/text()').get() @@ -62,7 +62,7 @@ more shortcuts: ``response.xpath()`` and ``response.css()``: >>> response.css('span::text').get() 'good' -Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class +Scrapy selectors are instances of :class:`~scrapy.Selector` class constructed by passing either :class:`~scrapy.http.TextResponse` object or markup as a string (in ``text`` argument). @@ -175,7 +175,7 @@ of ``None``: 'not-found' Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes -using ``.attrib`` property of a :class:`~scrapy.selector.Selector`: +using ``.attrib`` property of a :class:`~scrapy.Selector`: >>> [img.attrib['src'] for img in response.css('img')] ['image1_thumb.jpg', @@ -383,7 +383,7 @@ ID, or when selecting an unique element on a page): Using selectors with regular expressions ---------------------------------------- -:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting +:class:`~scrapy.Selector` also has a ``.re()`` method for extracting data using regular expressions. However, unlike using ``.xpath()`` or ``.css()`` methods, ``.re()`` returns a list of strings. So you can't construct nested ``.re()`` calls. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0b290598f..1290b4a5e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -67,7 +67,7 @@ Example:: Spiders (See the :ref:`topics-spiders` chapter for reference) can define their own settings that will take precedence and override the project ones. They can -do so by setting their :attr:`~scrapy.spiders.Spider.custom_settings` attribute:: +do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute:: class MySpider(scrapy.Spider): name = 'myspider' @@ -142,7 +142,7 @@ In a spider, the settings are available through ``self.settings``:: The ``settings`` attribute is set in the base Spider class after the spider is initialized. If you want to use the settings before the initialization (e.g., in your spider's ``__init__()`` method), you'll need to override the - :meth:`~scrapy.spiders.Spider.from_crawler` method. + :meth:`~scrapy.Spider.from_crawler` method. Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` attribute of the Crawler that is passed to ``from_crawler`` method in @@ -338,7 +338,7 @@ is non-zero, download delay is enforced per IP, not per domain. DEFAULT_ITEM_CLASS ------------------ -Default: ``'scrapy.item.Item'`` +Default: ``'scrapy.Item'`` The default class that will be used for instantiating items in the :ref:`the Scrapy shell `. @@ -360,7 +360,7 @@ The default headers used for Scrapy HTTP Requests. They're populated in the .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. .. setting:: DEPTH_LIMIT @@ -384,8 +384,8 @@ Default: ``0`` Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` -An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of -a :class:`~scrapy.http.Request` based on its depth. +An integer that is used to adjust the :attr:`~scrapy.Request.priority` of +a :class:`~scrapy.Request` based on its depth. The priority of a request is adjusted as follows:: @@ -816,14 +816,14 @@ The default (``RFPDupeFilter``) filters based on request fingerprint using the ``scrapy.utils.request.request_fingerprint`` function. In order to change the way duplicates are checked you could subclass ``RFPDupeFilter`` and override its ``request_fingerprint`` method. This method should accept -scrapy :class:`~scrapy.http.Request` object and return its fingerprint +scrapy :class:`~scrapy.Request` object and return its fingerprint (a string). You can disable filtering of duplicate requests by setting :setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``. Be very careful about this however, because you can get into crawling loops. It's usually a better idea to set the ``dont_filter`` parameter to -``True`` on the specific :class:`~scrapy.http.Request` that should not be +``True`` on the specific :class:`~scrapy.Request` that should not be filtered. .. setting:: DUPEFILTER_DEBUG diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index b910fc453..8c90a506c 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -118,7 +118,7 @@ Available Scrapy objects The Scrapy shell automatically creates some convenient objects from the downloaded page, like the :class:`~scrapy.http.Response` object and the -:class:`~scrapy.selector.Selector` objects (for both HTML and XML +:class:`~scrapy.Selector` objects (for both HTML and XML content). Those objects are: @@ -126,12 +126,12 @@ Those objects are: - ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object. - ``spider`` - the Spider which is known to handle the URL, or a - :class:`~scrapy.spiders.Spider` object if there is no spider found for the + :class:`~scrapy.Spider` object if there is no spider found for the current URL -- ``request`` - a :class:`~scrapy.http.Request` object of the last fetched +- ``request`` - a :class:`~scrapy.Request` object of the last fetched page. You can modify this request using - :meth:`~scrapy.http.Request.replace` or fetch a new request (without + :meth:`~scrapy.Request.replace` or fetch a new request (without leaving the shell) using the ``fetch`` shortcut. - ``response`` - a :class:`~scrapy.http.Response` object containing the last diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 530af1e37..a67cc1879 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -155,7 +155,7 @@ item_scraped :type item: :ref:`item object ` :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was scraped :type response: :class:`~scrapy.http.Response` object @@ -175,7 +175,7 @@ item_dropped :type item: :ref:`item object ` :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was dropped :type response: :class:`~scrapy.http.Response` object @@ -203,7 +203,7 @@ item_error :type response: :class:`~scrapy.http.Response` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param failure: the exception raised :type failure: twisted.python.failure.Failure @@ -223,7 +223,7 @@ spider_closed This signal supports returning deferreds from its handlers. :param spider: the spider which has been closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param reason: a string which describes the reason why the spider was closed. If it was closed because the spider has completed scraping, the reason @@ -247,7 +247,7 @@ spider_opened This signal supports returning deferreds from its handlers. :param spider: the spider which has been opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object spider_idle ~~~~~~~~~~~ @@ -278,7 +278,7 @@ spider_idle This signal does not support returning deferreds from its handlers. :param spider: the spider which has gone idle - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. note:: Scheduling some requests in your :signal:`spider_idle` handler does **not** guarantee that it can prevent the spider from being closed, @@ -303,7 +303,7 @@ spider_error :type response: :class:`~scrapy.http.Response` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object Request signals --------------- @@ -314,16 +314,16 @@ request_scheduled .. signal:: request_scheduled .. function:: request_scheduled(request, spider) - Sent when the engine schedules a :class:`~scrapy.http.Request`, to be + Sent when the engine schedules a :class:`~scrapy.Request`, to be downloaded later. This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object request_dropped ~~~~~~~~~~~~~~~ @@ -331,16 +331,16 @@ request_dropped .. signal:: request_dropped .. function:: request_dropped(request, spider) - Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be + Sent when a :class:`~scrapy.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object request_reached_downloader ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -348,15 +348,15 @@ request_reached_downloader .. signal:: request_reached_downloader .. function:: request_reached_downloader(request, spider) - Sent when a :class:`~scrapy.http.Request` reached downloader. + Sent when a :class:`~scrapy.Request` reached downloader. This signal does not support returning deferreds from its handlers. :param request: the request that reached downloader - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object request_left_downloader ~~~~~~~~~~~~~~~~~~~~~~~ @@ -366,16 +366,16 @@ request_left_downloader .. versionadded:: 2.0 - Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of + Sent when a :class:`~scrapy.Request` leaves the downloader, even in case of failure. This signal does not support returning deferreds from its handlers. :param request: the request that reached the downloader - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object bytes_received ~~~~~~~~~~~~~~ @@ -402,10 +402,10 @@ bytes_received :type data: :class:`bytes` object :param request: the request that generated the download - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider associated with the response - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object headers_received ~~~~~~~~~~~~~~~~ @@ -432,10 +432,10 @@ headers_received :type body_length: `int` :param request: the request that generated the download - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider associated with the response - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object Response signals ---------------- @@ -455,10 +455,10 @@ response_received :type response: :class:`~scrapy.http.Response` object :param request: the request that generated the response - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. note:: The ``request`` argument might not contain the original request that reached the downloader, if a :ref:`topics-downloader-middleware` modifies @@ -479,7 +479,7 @@ response_downloaded :type response: :class:`~scrapy.http.Response` object :param request: the request that generated the response - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 11bbbb58d..f0158dc41 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -93,7 +93,7 @@ object gives you access, for example, to the :ref:`settings `. :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_spider_output(response, result, spider) @@ -102,7 +102,7 @@ object gives you access, for example, to the :ref:`settings `. it has processed the response. :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.http.Request` objects and :ref:`item object + :class:`~scrapy.Request` objects and :ref:`item object `. :param response: the response which generated this output from the @@ -110,11 +110,11 @@ object gives you access, for example, to the :ref:`settings `. :type response: :class:`~scrapy.http.Response` object :param result: the result returned by the spider - :type result: an iterable of :class:`~scrapy.http.Request` objects and + :type result: an iterable of :class:`~scrapy.Request` objects and :ref:`item object ` :param spider: the spider whose result is being processed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_spider_exception(response, exception, spider) @@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Request` objects and :ref:`item object + iterable of :class:`~scrapy.Request` objects and :ref:`item object `. If it returns ``None``, Scrapy will continue processing this exception, @@ -142,7 +142,7 @@ object gives you access, for example, to the :ref:`settings `. :type exception: :exc:`Exception` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_start_requests(start_requests, spider) @@ -152,7 +152,7 @@ object gives you access, for example, to the :ref:`settings `. items). It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.http.Request` objects. + return another iterable of :class:`~scrapy.Request` objects. .. note:: When implementing this method in your spider middleware, you should always return an iterable (that follows the input one) and @@ -164,10 +164,10 @@ object gives you access, for example, to the :ref:`settings `. (like a time limit or item/page count). :param start_requests: the start requests - :type start_requests: an iterable of :class:`~scrapy.http.Request` + :type start_requests: an iterable of :class:`~scrapy.Request` :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: from_crawler(cls, crawler) @@ -251,7 +251,7 @@ this:: .. reqmeta:: handle_httpstatus_all The ``handle_httpstatus_list`` key of :attr:`Request.meta -` can also be used to specify which response codes to +` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` to ``True`` if you want to allow any response code for a request, and ``False`` to disable the effects of the ``handle_httpstatus_all`` key. @@ -295,7 +295,7 @@ OffsiteMiddleware Filters out Requests for URLs outside the domains covered by the spider. This middleware filters out every request whose host names aren't in the - spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute. + spider's :attr:`~scrapy.Spider.allowed_domains` attribute. All subdomains of any domain in the list are also allowed. E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` but not ``www2.example.com`` nor ``example.com``. @@ -313,10 +313,10 @@ OffsiteMiddleware will be printed (but only for the first request filtered). If the spider doesn't define an - :attr:`~scrapy.spiders.Spider.allowed_domains` attribute, or the + :attr:`~scrapy.Spider.allowed_domains` attribute, or the attribute is empty, the offsite middleware will allow all requests. - If the request has the :attr:`~scrapy.http.Request.dont_filter` attribute + If the request has the :attr:`~scrapy.Request.dont_filter` attribute set, the offsite middleware will allow the request even if its domain is not listed in allowed domains. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 903fbd383..67b9e2e0e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -17,15 +17,15 @@ For spiders, the scraping cycle goes through something like this: those requests. The first requests to perform are obtained by calling the - :meth:`~scrapy.spiders.Spider.start_requests` method which (by default) - generates :class:`~scrapy.http.Request` for the URLs specified in the - :attr:`~scrapy.spiders.Spider.start_urls` and the - :attr:`~scrapy.spiders.Spider.parse` method as callback function for the + :meth:`~scrapy.Spider.start_requests` method which (by default) + generates :class:`~scrapy.Request` for the URLs specified in the + :attr:`~scrapy.Spider.start_urls` and the + :attr:`~scrapy.Spider.parse` method as callback function for the Requests. 2. In the callback function, you parse the response (web page) and return :ref:`item objects `, - :class:`~scrapy.http.Request` objects, or an iterable of these objects. + :class:`~scrapy.Request` objects, or an iterable of these objects. Those Requests will also contain a callback (maybe the same) and will then be downloaded by Scrapy and then their response handled by the specified callback. @@ -50,7 +50,8 @@ We will talk about those types here. scrapy.Spider ============= -.. class:: Spider() +.. class:: scrapy.spiders.Spider() +.. class:: scrapy.Spider() This is the simplest spider, and the one from which every other spider must inherit (including spiders that come bundled with Scrapy, as well as spiders @@ -86,7 +87,7 @@ scrapy.Spider A list of URLs where the spider will begin to crawl from, when no particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data + listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data contained in the start URLs. .. attribute:: custom_settings @@ -179,7 +180,7 @@ scrapy.Spider the same requirements as the :class:`Spider` class. This method, as well as any other Request callback, must return an - iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects + iterable of :class:`~scrapy.Request` and/or :ref:`item objects `. :param response: the response to parse @@ -234,7 +235,7 @@ Return multiple Requests and items from a single callback:: yield scrapy.Request(response.urljoin(href), self.parse) Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; -to give data more structure you can use :class:`~scrapy.item.Item` objects:: +to give data more structure you can use :class:`~scrapy.Item` objects:: import scrapy from myproject.items import MyItem @@ -373,7 +374,7 @@ CrawlSpider This method is called for each response produced for the URLs in the spider's ``start_urls`` attribute. It allows to parse the initial responses and must return either an - :ref:`item object `, a :class:`~scrapy.http.Request` + :ref:`item object `, a :class:`~scrapy.Request` object, or an iterable containing any of them. Crawling rules @@ -383,7 +384,7 @@ Crawling rules ``link_extractor`` is a :ref:`Link Extractor ` object which defines how links will be extracted from each crawled page. Each produced link will - be used to generate a :class:`~scrapy.http.Request` object, which will contain the + be used to generate a :class:`~scrapy.Request` object, which will contain the link's text in its ``meta`` dictionary (under the ``link_text`` key). If omitted, a default link extractor created with no arguments will be used, resulting in all links being extracted. @@ -392,9 +393,9 @@ Crawling rules object with that name will be used) to be called for each link extracted with the specified link extractor. This callback receives a :class:`~scrapy.http.Response` as its first argument and must return either a single instance or an iterable of - :ref:`item objects ` and/or :class:`~scrapy.http.Request` objects + :ref:`item objects ` and/or :class:`~scrapy.Request` objects (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response` - object will contain the text of the link that produced the :class:`~scrapy.http.Request` + object will contain the text of the link that produced the :class:`~scrapy.Request` in its ``meta`` dictionary (under the ``link_text`` key) ``cb_kwargs`` is a dict containing the keyword arguments to be passed to the @@ -411,7 +412,7 @@ Crawling rules ``process_request`` is a callable (or a string, in which case a method from the spider object with that name will be used) which will be called for every - :class:`~scrapy.http.Request` extracted by this rule. This callable should + :class:`~scrapy.Request` extracted by this rule. This callable should take said request as first argument and the :class:`~scrapy.http.Response` from which the request originated as second argument. It must return a ``Request`` object or ``None`` (to filter out the request). @@ -470,7 +471,7 @@ Let's now take a look at an example CrawlSpider with rules:: This spider would start crawling example.com's home page, collecting category links, and item links, parsing the latter with the ``parse_item`` method. For each item response, some data will be extracted from the HTML using XPath, and -an :class:`~scrapy.item.Item` will be filled with it. +an :class:`~scrapy.Item` will be filled with it. XMLFeedSpider ------------- @@ -493,11 +494,11 @@ XMLFeedSpider - ``'iternodes'`` - a fast iterator based on regular expressions - - ``'html'`` - an iterator which uses :class:`~scrapy.selector.Selector`. + - ``'html'`` - an iterator which uses :class:`~scrapy.Selector`. Keep in mind this uses DOM parsing and must load all DOM in memory which could be a problem for big feeds - - ``'xml'`` - an iterator which uses :class:`~scrapy.selector.Selector`. + - ``'xml'`` - an iterator which uses :class:`~scrapy.Selector`. Keep in mind this uses DOM parsing and must load all DOM in memory which could be a problem for big feeds @@ -515,7 +516,7 @@ XMLFeedSpider available in that document that will be processed with this spider. The ``prefix`` and ``uri`` will be used to automatically register namespaces using the - :meth:`~scrapy.selector.Selector.register_namespace` method. + :meth:`~scrapy.Selector.register_namespace` method. You can then specify nodes with namespaces in the :attr:`itertag` attribute. @@ -542,10 +543,10 @@ XMLFeedSpider This method is called for the nodes matching the provided tag name (``itertag``). Receives the response and an - :class:`~scrapy.selector.Selector` for each node. Overriding this + :class:`~scrapy.Selector` for each node. Overriding this method is mandatory. Otherwise, you spider won't work. This method must return an :ref:`item object `, a - :class:`~scrapy.http.Request` object, or an iterable containing any of + :class:`~scrapy.Request` object, or an iterable containing any of them. .. method:: process_results(response, results) @@ -587,7 +588,7 @@ These spiders are pretty easy to use, let's have a look at one example:: Basically what we did up there was to create a spider that downloads a feed from the given ``start_urls``, and then iterates through each of its ``item`` tags, -prints them out, and stores some random data in an :class:`~scrapy.item.Item`. +prints them out, and stores some random data in an :class:`~scrapy.Item`. CSVFeedSpider ------------- From bcce0660573cf9309e3decf853488da2ef8bc576 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 14 Jul 2021 12:56:07 -0300 Subject: [PATCH 563/568] Update ItemFilter (#5203) --- scrapy/extensions/feedexport.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 84a79e32d..bd4808e2b 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,6 +11,7 @@ import sys import warnings from datetime import datetime from tempfile import NamedTemporaryFile +from typing import Any, Optional, Tuple from urllib.parse import unquote, urlparse from twisted.internet import defer, threads @@ -54,16 +55,19 @@ class ItemFilter: :param feed_options: feed specific options passed from FeedExporter :type feed_options: dict """ + feed_options: Optional[dict] + item_classes: Tuple - def __init__(self, feed_options): + def __init__(self, feed_options: Optional[dict]) -> None: self.feed_options = feed_options - self.item_classes = set() + if feed_options is not None: + self.item_classes = tuple( + load_object(item_class) for item_class in feed_options.get("item_classes") or () + ) + else: + self.item_classes = tuple() - if 'item_classes' in self.feed_options: - for item_class in self.feed_options['item_classes']: - self.item_classes.add(load_object(item_class)) - - def accepts(self, item): + def accepts(self, item: Any) -> bool: """ Return ``True`` if `item` should be exported or ``False`` otherwise. @@ -73,9 +77,8 @@ class ItemFilter: :rtype: bool """ if self.item_classes: - return isinstance(item, tuple(self.item_classes)) - - return True # accept all items if none declared in item_classes + return isinstance(item, self.item_classes) + return True # accept all items by default class IFeedStorage(Interface): From 89b654b82df763a59d6a37e92e796c1478c768ce Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 16 Jul 2021 15:18:14 +0500 Subject: [PATCH 564/568] Make the pylint test pass (#5207) Co-authored-by: Vostretsov Nikita --- .github/workflows/checks.yml | 4 ++++ .github/workflows/tests-ubuntu.yml | 4 ++++ pylintrc | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e7080db9a..6bdfcb5dc 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,6 +19,7 @@ jobs: - python-version: 3.8 env: TOXENV: pylint + TOX_PIP_VERSION: 20.3.3 - python-version: 3.9 env: TOXENV: typing @@ -37,5 +38,8 @@ jobs: - name: Run check env: ${{ matrix.env }} run: | + if [[ ! -z "$TOX_PIP_VERSION" ]]; then + pip install tox-pip-version + fi pip install -U tox tox diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index b42e8b127..521d7ae70 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -40,6 +40,7 @@ jobs: - python-version: 3.8 env: TOXENV: extra-deps + TOX_PIP_VERSION: 20.3.3 - python-version: 3.9 env: TOXENV: asyncio @@ -68,6 +69,9 @@ jobs: $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi + if [[ ! -z "$TOX_PIP_VERSION" ]]; then + pip install tox-pip-version + fi pip install -U tox tox diff --git a/pylintrc b/pylintrc index 972bf99de..a44712507 100644 --- a/pylintrc +++ b/pylintrc @@ -6,6 +6,7 @@ jobs=1 # >1 hides results disable=abstract-method, anomalous-backslash-in-string, arguments-differ, + arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, bad-continuation, @@ -21,6 +22,8 @@ disable=abstract-method, cell-var-from-loop, comparison-with-callable, consider-iterating-dictionary, + consider-using-dict-items, + consider-using-from-import, consider-using-in, consider-using-set-comprehension, consider-using-sys-exit, @@ -105,6 +108,7 @@ disable=abstract-method, unsubscriptable-object, unused-argument, unused-import, + unused-private-member, unused-variable, unused-wildcard-import, used-before-assignment, From ee2df97bbdf9120ccefc9c132bcbf0994479f948 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 16 Jul 2021 17:28:32 +0500 Subject: [PATCH 565/568] Pin the libxml2 version in CI as a newer one breaks lxml (#5208) --- .github/workflows/tests-ubuntu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 521d7ae70..57188bd63 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -57,7 +57,8 @@ jobs: if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update - sudo apt-get install libxml2-dev libxslt-dev + # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version + sudo apt-get install libxml2-dev/bionic-updates libxslt-dev - name: Run tests env: ${{ matrix.env }} From 70dddfe2b293171db4c58511175edf55bbe1831c Mon Sep 17 00:00:00 2001 From: Pascal Corpet Date: Wed, 21 Jul 2021 17:10:10 +0200 Subject: [PATCH 566/568] Typing: switch to a newer version of MyPy to check types --- tests/CrawlerProcess/asyncio_deferred_signal.py | 2 ++ tox.ini | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index 46c2a12a4..bdd3c1fef 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -1,5 +1,6 @@ import asyncio import sys +from typing import Optional from scrapy import Spider from scrapy.crawler import CrawlerProcess @@ -31,6 +32,7 @@ class UrlSpider(Spider): if __name__ == "__main__": + ASYNCIO_EVENT_LOOP: Optional[str] try: ASYNCIO_EVENT_LOOP = sys.argv[1] except IndexError: diff --git a/tox.ini b/tox.ini index 8167aff96..4c4bbff6e 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,10 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==0.780 + lxml-stubs==0.2.0 + mypy==0.910 + types-pyOpenSSL==20.0.3 + types-setuptools==57.0.0 commands = mypy --show-error-codes {posargs: scrapy tests} From b22a0043988a4f3c54709988de99f489db44f78d Mon Sep 17 00:00:00 2001 From: Rob Banagale Date: Mon, 26 Jul 2021 11:51:32 -0700 Subject: [PATCH 567/568] Document media pipeline file naming (#5152) --- docs/topics/media-pipeline.rst | 88 ++++++++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 14 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 3438cb637..46bd2859b 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -111,25 +111,82 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting:: IMAGES_STORE = '/path/to/valid/dir' +.. _topics-file-naming: + +File Naming +=========== + +Default File Naming +------------------- + +By default, files are stored using an `SHA-1 hash`_ of their URLs for the file names. + +For example, the following image URL:: + + http://www.example.com/image.jpg + +Whose ``SHA-1 hash`` is:: + + 3afec3b4765f8f0a07b78f98c07b83f013567a0a + +Will be downloaded and stored using your chosen :ref:`storage method ` and the following file name:: + + 3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg + +Custom File Naming +------------------- + +You may wish to use a different calculated file name for saved files. +For example, classifying an image by including meta in the file name. + +Customize file names by overriding the ``file_path`` method of your +media pipeline. + +For example, an image pipeline with image URL:: + + http://www.example.com/product/images/large/front/0000000004166 + +Can be processed into a file name with a condensed hash and the perspective +``front``:: + + 00b08510e4_front.jpg + +By overriding ``file_path`` like this: + +.. code-block:: python + + import hashlib + from os.path import splitext + + def file_path(self, request, response=None, info=None, *, item=None): + image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5) + image_perspective = request.url.split('/')[-2] + image_filename = f'{image_url_hash}_{image_perspective}.jpg' + + return image_filename + +.. warning:: + If your custom file name scheme relies on meta data that can vary between + scrapes it may lead to unexpected re-downloading of existing media using + new file names. + + For example, if your custom file name scheme uses a product title and the + site changes an item's product title between scrapes, Scrapy will re-download + the same media using updated file names. + +For more information about the ``file_path`` method, see :ref:`topics-media-pipeline-override`. + +.. _topics-supported-storage: + Supported Storage ================= File system storage ------------------- -The files are stored using a `SHA1 hash`_ of their URLs for the file names. +File system storage will save files to the following path:: -For example, the following image URL:: - - http://www.example.com/image.jpg - -Whose ``SHA1 hash`` is:: - - 3afec3b4765f8f0a07b78f98c07b83f013567a0a - -Will be downloaded and stored in the following file:: - - /full/3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg + /full/ Where: @@ -139,6 +196,9 @@ Where: * ``full`` is a sub-directory to separate full images from thumbnails (if used). For more info see :ref:`topics-images-thumbnails`. +* ```` is the file name assigned to the file. For more info see :ref:`topics-file-naming`. + + .. _media-pipeline-ftp: FTP server storage @@ -353,9 +413,9 @@ Where: * ```` is the one specified in the :setting:`IMAGES_THUMBS` dictionary keys (``small``, ``big``, etc) -* ```` is the `SHA1 hash`_ of the image url +* ```` is the `SHA-1 hash`_ of the image url -.. _SHA1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions +.. _SHA-1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions Example of image files stored using ``small`` and ``big`` thumbnail names:: From abe0b37d307d40897ca6d7e61aa5c137c8e6a4c1 Mon Sep 17 00:00:00 2001 From: laggardkernel Date: Tue, 27 Jul 2021 17:11:32 +0800 Subject: [PATCH 568/568] Cleanup leftover boto2 code in S3DownloaderHandler (#5209) S3DownloaderHandler.conn is a leftover attribute from 5e99758. --- scrapy/core/downloader/handlers/s3.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 1966570d4..31f1be31a 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,5 +1,3 @@ -from urllib.parse import unquote - from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore_available @@ -59,7 +57,7 @@ class S3DownloadHandler: url = f'{scheme}://{bucket}.s3.amazonaws.com{path}' if self.anon: request = request.replace(url=url) - elif self._signer is not None: + else: import botocore.awsrequest awsrequest = botocore.awsrequest.AWSRequest( method=request.method, @@ -69,14 +67,4 @@ class S3DownloadHandler: self._signer.add_auth(awsrequest) request = request.replace( url=url, headers=awsrequest.headers.items()) - else: - signed_headers = self.conn.make_request( - method=request.method, - bucket=bucket, - key=unquote(p.path), - query_args=unquote(p.query), - headers=request.headers, - data=request.body, - ) - request = request.replace(url=url, headers=signed_headers) return self._download_http(request, spider)