From 2a540206a74af8d38a01aaa5a37adc1008cad6ca Mon Sep 17 00:00:00 2001 From: nramirezuy Date: Tue, 19 Aug 2014 13:57:00 -0300 Subject: [PATCH 01/80] fix xmliter namespace on selected node --- scrapy/utils/iterators.py | 22 +++++++++++++++------- tests/test_utils_iterators.py | 6 +++++- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 150b077ae..11b873f2e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -20,19 +20,27 @@ def xmliter(obj, nodename): - a unicode string - a string encoded as utf-8 """ - HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S) + DOCUMENT_HEADER_RE = re.compile(r'<\?xml[^>]+>\s*', re.S) HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S) + END_TAG_RE = re.compile(r'<\s*/([^\s>]+)\s*>', re.S) + NAMESPACE_RE = re.compile(r'((xmlns[:A-Za-z]*)=[^>\s]+)', re.S) text = _body_or_str(obj) - header_start = re.search(HEADER_START_RE, text) - header_start = header_start.group(1).strip() if header_start else '' - header_end = re_rsearch(HEADER_END_RE, text) - header_end = text[header_end[1]:].strip() if header_end else '' + document_header = re.search(DOCUMENT_HEADER_RE, text) + document_header = document_header.group().strip() if document_header else '' + header_end_idx = re_rsearch(HEADER_END_RE, text) + header_end = text[header_end_idx[1]:].strip() if header_end_idx else '' + namespaces = {} + if header_end: + for tagname in reversed(re.findall(END_TAG_RE, header_end)): + tag = re.search(r'<\s*%s.*?xmlns[:=][^>]*>' % tagname, text[:header_end_idx[1]], re.S) + if tag: + namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group())) r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) for match in r.finditer(text): - nodetext = header_start + match.group() + header_end - yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] + nodetext = document_header + match.group().replace(nodename, '%s %s' % (nodename, ' '.join(namespaces.values())), 1) + header_end + yield Selector(text=nodetext, type='xml') def csviter(obj, delimiter=None, headers=None, encoding=None): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index fe53f831f..8b5941605 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -61,7 +61,6 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url='http://mydummycompany.com', body=body) my_iter = self.xmliter(response, 'item') - node = next(my_iter) node.register_namespace('g', 'http://base.google.com/ns/1.0') self.assertEqual(node.xpath('title/text()').extract(), ['Item 1']) @@ -74,6 +73,11 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('id/text()').extract(), []) self.assertEqual(node.xpath('price/text()').extract(), []) + my_iter = self.xmliter(response, 'g:image_link') + node = next(my_iter) + node.register_namespace('g', 'http://base.google.com/ns/1.0') + self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + def test_xmliter_exception(self): body = u"""onetwo""" From e90be0d8a5e3c20ab6aced22ccac59a876db8c99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 21 Aug 2020 14:09:52 +0200 Subject: [PATCH 02/80] Mark the new test as xfail for xmliter_lxml --- tests/test_utils_iterators.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index bbdc88dd1..ae64e36cf 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,5 +1,6 @@ import os +from pytest import mark from twisted.trial import unittest from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml @@ -149,6 +150,26 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('id/text()').getall(), []) self.assertEqual(node.xpath('price/text()').getall(), []) + def test_xmliter_namespaced_nodename(self): + body = b""" + + + + My Dummy Company + http://www.mydummycompany.com + This is a dummy company. We do nothing. + + Item 1 + This is item 1 + http://www.mydummycompany.com/items/1 + http://www.mydummycompany.com/images/item1.jpg + ITEM_1 + 400 + + + + """ + response = XmlResponse(url='http://mydummycompany.com', body=body) my_iter = self.xmliter(response, 'g:image_link') node = next(my_iter) node.register_namespace('g', 'http://base.google.com/ns/1.0') @@ -187,6 +208,10 @@ class XmliterTestCase(unittest.TestCase): class LxmlXmliterTestCase(XmliterTestCase): xmliter = staticmethod(xmliter_lxml) + @mark.xfail(reason='known bug of the current implementation') + def test_xmliter_namespaced_nodename(self): + super().test_xmliter_namespaced_nodename() + def test_xmliter_iterate_namespace(self): body = b""" From afd3a4d116809ecf4c334657c5150c157aa9465b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 21 Aug 2020 17:04:02 +0200 Subject: [PATCH 03/80] Fix style issue --- scrapy/utils/iterators.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c356ad7f8..6f6b5e337 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -41,7 +41,15 @@ def xmliter(obj, nodename): r = re.compile(r'<%(np)s[\s>].*?' % {'np': nodename_patt}, re.DOTALL) for match in r.finditer(text): - nodetext = document_header + match.group().replace(nodename, '%s %s' % (nodename, ' '.join(namespaces.values())), 1) + header_end + nodetext = ( + document_header + + match.group().replace( + nodename, + '%s %s' % (nodename, ' '.join(namespaces.values())), + 1 + ) + + header_end + ) yield Selector(text=nodetext, type='xml') From 2f28cee3ce482a9380c816b689fc6a5dabc60295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 25 Aug 2020 17:49:17 +0200 Subject: [PATCH 04/80] Add a test to cover searching for a missing node name --- tests/test_utils_iterators.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ae64e36cf..2adccebb8 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -175,6 +175,30 @@ class XmliterTestCase(unittest.TestCase): node.register_namespace('g', 'http://base.google.com/ns/1.0') self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + def test_xmliter_namespaced_nodename_missing(self): + body = b""" + + + + My Dummy Company + http://www.mydummycompany.com + This is a dummy company. We do nothing. + + Item 1 + This is item 1 + http://www.mydummycompany.com/items/1 + http://www.mydummycompany.com/images/item1.jpg + ITEM_1 + 400 + + + + """ + response = XmlResponse(url='http://mydummycompany.com', body=body) + my_iter = self.xmliter(response, 'g:link_image') + with self.assertRaises(StopIteration): + next(my_iter) + def test_xmliter_exception(self): body = ( '' From a8e895e684184e967a751ea28a7102f21dd74834 Mon Sep 17 00:00:00 2001 From: maranqz Date: Sun, 30 Aug 2020 10:57:22 +0300 Subject: [PATCH 05/80] kwargs for Item exporters classes test docs --- docs/topics/feed-exports.rst | 5 +++++ scrapy/extensions/feedexport.py | 1 + scrapy/utils/conf.py | 1 + tests/test_feedexport.py | 37 +++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 9fb2189e8..e69a64195 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -303,6 +303,9 @@ For instance:: 'store_empty': False, 'fields': None, 'indent': 4, + 'item_export_kwargs': { + 'export_empty_fields': True, + }, }, '/home/user/documents/items.xml': { 'format': 'xml', @@ -332,6 +335,8 @@ as a fallback value if that key is not provided for a specific feed definition: - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. +- ``item_export_kwargs``: dict with kwargs for :ref:`Item exporters ` classes. + - ``overwrite``: whether to overwrite the file if it already exists (``True``) or append to its content (``False``). diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 980825499..f32d48fa5 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -360,6 +360,7 @@ class FeedExporter: fields_to_export=feed_options['fields'], encoding=feed_options['encoding'], indent=feed_options['indent'], + **feed_options['item_export_kwargs'], ) slot = _FeedSlot( file=file, diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 4e7a9967e..b904c4a03 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -121,6 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings): out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) + out.setdefault("item_export_kwargs", dict()) if settings["FEED_EXPORT_INDENT"] is None: out.setdefault("indent", None) else: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 840e0f87b..e88e4f5ce 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1252,6 +1252,43 @@ class FeedExportTest(FeedExportTestBase): for fmt in ['json', 'xml', 'csv']: self.assertIn(f'Error storing {fmt} feed (2 items)', str(log)) + @defer.inlineCallbacks + def test_extend_kwargs(self): + items = [{'foo': 'FOO', 'bar': 'BAR'}] + + expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8') + expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8') + test_cases = [ + # with title + { + 'options': { + 'format': 'csv', + 'item_export_kwargs': dict(include_headers_line=True), + }, + 'expected': expected_with_title_csv, + }, + # without title + { + 'options': { + 'format': 'csv', + 'item_export_kwargs': dict(include_headers_line=False), + }, + 'expected': expected_without_title_csv, + }, + ] + + for row in test_cases: + feed_options = row['options'] + settings = { + 'FEEDS': { + self._random_temp_filename(): feed_options, + }, + 'FEED_EXPORT_INDENT': None, + } + + data = yield self.exported_data(items, settings) + self.assertEqual(row['expected'], data[feed_options['format']]) + class BatchDeliveriesTest(FeedExportTestBase): __test__ = True From fc3c66ce950e945e7ac6e19fea31e8454147ac29 Mon Sep 17 00:00:00 2001 From: maranqz Date: Sun, 30 Aug 2020 11:44:48 +0300 Subject: [PATCH 06/80] fix tests: * FeedExportConfigTestCase.test_feed_complete_default_values_from_settings_empty * FeedExportConfigTestCase.test_feed_complete_default_values_from_settings_non_empty --- tests/test_utils_conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 061bc8c7c..dc2560add 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -176,6 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": (1, 2, 3, 4), "batch_item_count": 2, + "item_export_kwargs": dict(), }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -198,6 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": None, "batch_item_count": 2, + "item_export_kwargs": dict(), }) From 71d2c2f1a319b570eb2026707d49c4f62d59296e Mon Sep 17 00:00:00 2001 From: maranqz Date: Sun, 30 Aug 2020 12:43:44 +0300 Subject: [PATCH 07/80] improve view of dict --- tests/test_feedexport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e88e4f5ce..5738e36f1 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1263,7 +1263,7 @@ class FeedExportTest(FeedExportTestBase): { 'options': { 'format': 'csv', - 'item_export_kwargs': dict(include_headers_line=True), + 'item_export_kwargs': {'include_headers_line': True}, }, 'expected': expected_with_title_csv, }, @@ -1271,7 +1271,7 @@ class FeedExportTest(FeedExportTestBase): { 'options': { 'format': 'csv', - 'item_export_kwargs': dict(include_headers_line=False), + 'item_export_kwargs': {'include_headers_line': False}, }, 'expected': expected_without_title_csv, }, From d10464ca96a24d37c854992e2485d622e8eec2b6 Mon Sep 17 00:00:00 2001 From: Ilia Sergunin Date: Tue, 1 Sep 2020 10:13:40 +0300 Subject: [PATCH 08/80] Update docs/topics/feed-exports.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index e69a64195..1744cfd74 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -335,7 +335,7 @@ as a fallback value if that key is not provided for a specific feed definition: - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. -- ``item_export_kwargs``: dict with kwargs for :ref:`Item exporters ` classes. +- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. - ``overwrite``: whether to overwrite the file if it already exists (``True``) or append to its content (``False``). From 959222df7e80334810636166a8adb60ef35728de Mon Sep 17 00:00:00 2001 From: drs-11 Date: Sat, 5 Sep 2020 21:32:05 +0530 Subject: [PATCH 09/80] check for unparseable no_proxy values --- scrapy/downloadermiddlewares/httpproxy.py | 7 ++++++- tests/test_downloadermiddleware_httpproxy.py | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 04da11311..d2665b655 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -13,7 +13,12 @@ class HttpProxyMiddleware: self.auth_encoding = auth_encoding self.proxies = {} for type_, url in getproxies().items(): - self.proxies[type_] = self._get_proxy(url, type_) + try: + self.proxies[type_] = self._get_proxy(url, type_) + # some values such as '/var/run/docker.sock' can't be parsed + # by _parse_proxy and as such should be skipped + except ValueError: + continue @classmethod def from_crawler(cls, crawler): diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 351631eb8..81d6cc335 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -123,7 +123,11 @@ class TestHttpProxyMiddleware(TestCase): def test_no_proxy(self): os.environ['http_proxy'] = 'https://proxy.for.http:3128' + os.environ['no_proxy'] = '/var/run/docker.sock' mw = HttpProxyMiddleware() + # '/var/run/docker.sock' may be used by the user for + # no_proxy value but is not parseable and should be skipped + assert 'no' not in mw.proxies os.environ['no_proxy'] = '*' req = Request('http://noproxy.com') From 82ba7c8b529e004a62db4681f16a482ae4e769f1 Mon Sep 17 00:00:00 2001 From: drs-11 Date: Thu, 10 Sep 2020 20:56:39 +0530 Subject: [PATCH 10/80] created separate test for invalid no-proxy values --- tests/test_downloadermiddleware_httpproxy.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 81d6cc335..7c97bf32a 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -123,11 +123,7 @@ class TestHttpProxyMiddleware(TestCase): def test_no_proxy(self): os.environ['http_proxy'] = 'https://proxy.for.http:3128' - os.environ['no_proxy'] = '/var/run/docker.sock' mw = HttpProxyMiddleware() - # '/var/run/docker.sock' may be used by the user for - # no_proxy value but is not parseable and should be skipped - assert 'no' not in mw.proxies os.environ['no_proxy'] = '*' req = Request('http://noproxy.com') @@ -149,3 +145,10 @@ class TestHttpProxyMiddleware(TestCase): req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'}) assert mw.process_request(req, spider) is None self.assertEqual(req.meta, {'proxy': 'http://proxy.com'}) + + def test_no_proxy_invalid_values(self): + os.environ['no_proxy'] = '/var/run/docker.sock' + mw = HttpProxyMiddleware() + # '/var/run/docker.sock' may be used by the user for + # no_proxy value but is not parseable and should be skipped + assert 'no' not in mw.proxies From 008cf1c75ebe72a607149d7efd8f902f8763bc52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 20:45:21 +0200 Subject: [PATCH 11/80] Remove a test that has never been executed in Python 3 --- tests/test_downloader_handlers.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0a374c161..3e8d7e6b9 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -868,29 +868,6 @@ class S3TestCase(unittest.TestCase): self.assertEqual(httpreq.headers['Authorization'], b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') - def test_request_signing5(self): - try: - import botocore # noqa: F401 - except ImportError: - pass - else: - raise unittest.SkipTest( - 'botocore does not support overriding date with x-amz-date') - # deletes an object from the 'johnsmith' bucket using the - # path-style and Date alternative. - date = 'Tue, 27 Mar 2007 21:20:27 +0000' - req = Request( - 's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={ - 'Date': date, - 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000', - }) - with self._mocked_date(date): - httpreq = self.download_request(req, self.spider) - # botocore does not override Date with x-amz-date - self.assertEqual( - httpreq.headers['Authorization'], - b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') - def test_request_signing6(self): # uploads an object to a CNAME style virtual hosted bucket with metadata. date = 'Tue, 27 Mar 2007 21:06:08 +0000' From 56f05fb16476b40f773edc9e93d41c8893a349c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 22:01:09 +0200 Subject: [PATCH 12/80] Use mocking for tests/test_feedexport.py::S3FeedStorageTest::test_store --- tests/test_feedexport.py | 50 +++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 33ac51712..a8c0cf686 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -13,6 +13,7 @@ from logging import getLogger from pathlib import Path from string import ascii_letters, digits from unittest import mock +from unittest.mock import call from urllib.parse import urljoin, urlparse, quote from urllib.request import pathname2url @@ -254,21 +255,42 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): - assert_aws_environ() - uri = os.environ.get('S3_TEST_FILE_URI') - if not uri: - raise unittest.SkipTest("No S3 URI available for testing") - access_key = os.environ.get('AWS_ACCESS_KEY_ID') - secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') - storage = S3FeedStorage(uri, access_key, secret_key) + skip_if_no_boto() + + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + } + crawler = get_crawler(settings_dict=settings) + bucket = 'mybucket' + key = 'export.csv' + storage = S3FeedStorage.from_crawler(crawler, f's3://{bucket}/{key}') verifyObject(IFeedStorage, storage) - file = storage.open(scrapy.Spider("default")) - expected_content = b"content: \xe2\x98\x83" - file.write(expected_content) - yield storage.store(file) - u = urlparse(uri) - content = get_s3_content_and_delete(u.hostname, u.path[1:]) - self.assertEqual(content, expected_content) + + file = mock.MagicMock() + from botocore.stub import Stubber + with Stubber(storage.s3_client) as stub: + stub.add_response( + 'put_object', + expected_params={ + 'Body': file, + 'Bucket': bucket, + 'Key': key, + }, + service_response={}, + ) + + yield storage.store(file) + + stub.assert_no_pending_responses() + self.assertEqual( + file.method_calls, + [ + call.seek(0), + # The call to read does not happen with Stubber + call.close(), + ] + ) def test_init_without_acl(self): storage = S3FeedStorage( From 17e135377a564152e82cd2ad177a6a2749c15832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 22:54:39 +0200 Subject: [PATCH 13/80] Use mocking for tests/test_feedexport.py::BatchDeliveriesTest::test_s3_export --- tests/test_feedexport.py | 99 +++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 51 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index a8c0cf686..ac0b3ccaa 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -13,7 +13,6 @@ from logging import getLogger from pathlib import Path from string import ascii_letters, digits from unittest import mock -from unittest.mock import call from urllib.parse import urljoin, urlparse, quote from urllib.request import pathname2url @@ -42,7 +41,6 @@ from scrapy.extensions.feedexport import ( from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import ( - assert_aws_environ, get_s3_content_and_delete, get_crawler, mock_google_cloud_storage, @@ -286,9 +284,9 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual( file.method_calls, [ - call.seek(0), + mock.call.seek(0), # The call to read does not happen with Stubber - call.close(), + mock.call.close(), ] ) @@ -1518,46 +1516,53 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_s3_export(self): - """ - Test export of items into s3 bucket. - S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini - to perform this test: - [testenv] - setenv = - AWS_SECRET_ACCESS_KEY = ABCD - AWS_ACCESS_KEY_ID = EFGH - S3_TEST_BUCKET_NAME = IJKL - """ - try: - import boto3 - except ImportError: - raise unittest.SkipTest("S3FeedStorage requires boto3") + skip_if_no_boto() - assert_aws_environ() - s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME') - access_key = os.environ.get('AWS_ACCESS_KEY_ID') - secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') - if not s3_test_bucket_name: - raise unittest.SkipTest("No S3 BUCKET available for testing") - - chars = [random.choice(ascii_letters + digits) for _ in range(15)] - filename = ''.join(chars) - prefix = f'tmp/{filename}' - s3_test_file_uri = f's3://{s3_test_bucket_name}/{prefix}/%(batch_time)s.json' - storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) - settings = Settings({ - 'FEEDS': { - s3_test_file_uri: { - 'format': 'json', - }, - }, - 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, - }) + bucket = 'mybucket' items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), ] + + class CustomS3FeedStorage(S3FeedStorage): + + stubs = [] + + def open(self, *args, **kwargs): + from botocore.stub import ANY, Stubber + stub = Stubber(self.s3_client) + stub.activate() + CustomS3FeedStorage.stubs.append(stub) + stub.add_response( + 'put_object', + expected_params={ + 'Body': ANY, + 'Bucket': bucket, + 'Key': ANY, + }, + service_response={}, + ) + return super().open(*args, **kwargs) + + key = 'export.csv' + uri = f's3://{bucket}/{key}/%(batch_time)s.json' + batch_item_count = 1 + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'FEED_EXPORT_BATCH_ITEM_COUNT': batch_item_count, + 'FEED_STORAGES': { + 's3': CustomS3FeedStorage, + }, + 'FEEDS': { + uri: { + 'format': 'json', + }, + }, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, uri) verifyObject(IFeedStorage, storage) class TestSpider(scrapy.Spider): @@ -1567,22 +1572,14 @@ class BatchDeliveriesTest(FeedExportTestBase): for item in items: yield item - s3 = boto3.resource('s3') - my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') - - with MockServer() as s: + with MockServer() as server: runner = CrawlerRunner(Settings(settings)) - TestSpider.start_urls = [s.url('/')] + TestSpider.start_urls = [server.url('/')] yield runner.crawl(TestSpider) - for file_uri in my_bucket.objects.filter(Prefix=prefix): - content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key) - if not content and not items: - break - content = json.loads(content.decode('utf-8')) - expected_batch, items = items[:batch_size], items[batch_size:] - self.assertEqual(expected_batch, content) + self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)+1) + for stub in CustomS3FeedStorage.stubs[:-1]: + stub.assert_no_pending_responses() class FeedExportInitTest(unittest.TestCase): From 35726da434a9bc61dcb1ad5ef475c7d030023c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 22:55:25 +0200 Subject: [PATCH 14/80] tests/test_feedexport.py: remove unused import --- tests/test_feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ac0b3ccaa..18f7f8458 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -13,7 +13,7 @@ from logging import getLogger from pathlib import Path from string import ascii_letters, digits from unittest import mock -from urllib.parse import urljoin, urlparse, quote +from urllib.parse import urljoin, quote from urllib.request import pathname2url import lxml.etree From c3b740f07814107e643bb0d073ea116049e37cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 23:25:37 +0200 Subject: [PATCH 15/80] Use mocking for tests/test_pipeline_files.py::TestS3FilesStore::test_persist --- scrapy/utils/test.py | 9 ---- tests/test_pipeline_files.py | 100 +++++++++++++++++++++++++++-------- 2 files changed, 78 insertions(+), 31 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 94d0ae2d3..cf251442f 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -13,15 +13,6 @@ from twisted.trial.unittest import SkipTest from scrapy.utils.boto import is_botocore_available -def assert_aws_environ(): - """Asserts the current environment is suitable for running AWS testsi. - Raises SkipTest with the reason if it's not. - """ - skip_if_no_boto() - if 'AWS_ACCESS_KEY_ID' not in os.environ: - raise SkipTest("AWS keys not found") - - def assert_gcs_environ(): if 'GCS_PROJECT_ID' not in os.environ: raise SkipTest("GCS_PROJECT_ID not found") diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index d5b0bb3d8..1b14e3b1e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,6 +1,7 @@ import os import random import time +from datetime import datetime from io import BytesIO from shutil import rmtree from tempfile import mkdtemp @@ -23,11 +24,11 @@ from scrapy.pipelines.files import ( ) from scrapy.settings import Settings from scrapy.utils.test import ( - assert_aws_environ, assert_gcs_environ, get_ftp_content_and_delete, get_gcs_content_and_delete, get_s3_content_and_delete, + skip_if_no_boto, ) @@ -414,32 +415,87 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): class TestS3FilesStore(unittest.TestCase): + @defer.inlineCallbacks def test_persist(self): - assert_aws_environ() - uri = os.environ.get('S3_TEST_FILE_URI') - if not uri: - raise unittest.SkipTest("No S3 URI available for testing") - data = b"TestS3FilesStore: \xe2\x98\x83" - buf = BytesIO(data) + skip_if_no_boto() + + bucket = 'mybucket' + key = 'export.csv' + uri = f's3://{bucket}/{key}' + buffer = mock.MagicMock() meta = {'foo': 'bar'} path = '' + content_type = 'image/png' + store = S3FilesStore(uri) - yield store.persist_file( - path, buf, info=None, meta=meta, - headers={'Content-Type': 'image/png'}) - s = yield store.stat_file(path, info=None) - self.assertIn('last_modified', s) - self.assertIn('checksum', s) - self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8') - u = urlparse(uri) - content, key = get_s3_content_and_delete( - u.hostname, u.path[1:], with_key=True) - self.assertEqual(content, data) - self.assertEqual(key['Metadata'], {'foo': 'bar'}) - self.assertEqual( - key['CacheControl'], S3FilesStore.HEADERS['Cache-Control']) - self.assertEqual(key['ContentType'], 'image/png') + from botocore.stub import Stubber + with Stubber(store.s3_client) as stub: + stub.add_response( + 'put_object', + expected_params={ + 'ACL': S3FilesStore.POLICY, + 'Body': buffer, + 'Bucket': bucket, + 'CacheControl': S3FilesStore.HEADERS['Cache-Control'], + 'ContentType': content_type, + 'Key': key, + 'Metadata': meta, + }, + service_response={}, + ) + + yield store.persist_file( + path, + buffer, + info=None, + meta=meta, + headers={'Content-Type': content_type}, + ) + + stub.assert_no_pending_responses() + self.assertEqual( + buffer.method_calls, + [ + mock.call.seek(0), + # The call to read does not happen with Stubber + ] + ) + + @defer.inlineCallbacks + def test_stat(self): + skip_if_no_boto() + + bucket = 'mybucket' + key = 'export.csv' + uri = f's3://{bucket}/{key}' + checksum = '3187896a9657a28163abb31667df64c8' + + store = S3FilesStore(uri) + from botocore.stub import Stubber + with Stubber(store.s3_client) as stub: + stub.add_response( + 'head_object', + expected_params={ + 'Bucket': bucket, + 'Key': key, + }, + service_response={ + 'ETag': f'"{checksum}"', + 'LastModified': datetime(2019, 12, 1), + }, + ) + + file_stats = yield store.stat_file('', info=None) + self.assertEqual( + file_stats, + { + 'checksum': checksum, + 'last_modified': 1575154800, + }, + ) + + stub.assert_no_pending_responses() class TestGCSFilesStore(unittest.TestCase): From 8f46e845190b7affb7a02c6842b465f0c5c3b76c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 23:28:16 +0200 Subject: [PATCH 16/80] Fix style issues --- scrapy/utils/test.py | 12 ------------ tests/test_feedexport.py | 3 +-- tests/test_pipeline_files.py | 1 - 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index cf251442f..24c38283a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -23,18 +23,6 @@ def skip_if_no_boto(): raise SkipTest('missing botocore library') -def get_s3_content_and_delete(bucket, path, with_key=False): - """ Get content from s3 key, and delete key afterwards. - """ - import botocore.session - session = botocore.session.get_session() - client = session.create_client('s3') - key = client.get_object(Bucket=bucket, Key=path) - content = key['Body'].read() - client.delete_object(Bucket=bucket, Key=path) - return (content, key) if with_key else content - - def get_gcs_content_and_delete(bucket, path): from google.cloud import storage client = storage.Client(project=os.environ.get('GCS_PROJECT_ID')) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 18f7f8458..3ed35bec5 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -41,7 +41,6 @@ from scrapy.extensions.feedexport import ( from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import ( - get_s3_content_and_delete, get_crawler, mock_google_cloud_storage, skip_if_no_boto, @@ -1577,7 +1576,7 @@ class BatchDeliveriesTest(FeedExportTestBase): TestSpider.start_urls = [server.url('/')] yield runner.crawl(TestSpider) - self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)+1) + self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1) for stub in CustomS3FeedStorage.stubs[:-1]: stub.assert_no_pending_responses() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 1b14e3b1e..e840298d9 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -27,7 +27,6 @@ from scrapy.utils.test import ( assert_gcs_environ, get_ftp_content_and_delete, get_gcs_content_and_delete, - get_s3_content_and_delete, skip_if_no_boto, ) From 07c1d9c25b6f499fbe2cc0f0139d78cf6c1d204a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 23:32:55 +0200 Subject: [PATCH 17/80] Remove boto3 as a dependency for tests --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 12e40295c..ea356c56a 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,6 @@ deps = -ctests/constraints.txt -rtests/requirements-py3.txt # Extras - boto3>=1.13.0 botocore>=1.4.87 Pillow>=4.0.0 passenv = From 6ef7c44061f74281e7a977c9b1de4892cf84caf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Sep 2020 12:45:21 +0200 Subject: [PATCH 18/80] Fix timezone test issue --- tests/test_pipeline_files.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e840298d9..4e1b90787 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -469,6 +469,7 @@ class TestS3FilesStore(unittest.TestCase): key = 'export.csv' uri = f's3://{bucket}/{key}' checksum = '3187896a9657a28163abb31667df64c8' + last_modified = datetime(2019, 12, 1) store = S3FilesStore(uri) from botocore.stub import Stubber @@ -481,7 +482,7 @@ class TestS3FilesStore(unittest.TestCase): }, service_response={ 'ETag': f'"{checksum}"', - 'LastModified': datetime(2019, 12, 1), + 'LastModified': last_modified, }, ) @@ -490,7 +491,7 @@ class TestS3FilesStore(unittest.TestCase): file_stats, { 'checksum': checksum, - 'last_modified': 1575154800, + 'last_modified': last_modified.timestamp(), }, ) From 894b509d7a4262cfc7548d78e9ff4831c6551c34 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 29 Sep 2020 23:37:28 -0300 Subject: [PATCH 19/80] Crawl rule: remove deprecated code Remove the compatibility layer that handles 'process_request' methods that do not receive a 'response' parameter --- scrapy/spiders/crawl.py | 34 ++++++++++--------------------- tests/test_spider.py | 45 +++++++++++++++++------------------------ 2 files changed, 30 insertions(+), 49 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index bc4551a54..1dcf2e6ab 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,14 +6,11 @@ See documentation in docs/topics/spiders.rst """ import copy -import warnings from typing import Sequence -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider -from scrapy.utils.python import get_func_args from scrapy.utils.spider import iterate_spider_output @@ -37,15 +34,22 @@ _default_link_extractor = LinkExtractor() class Rule: - def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, - process_links=None, process_request=None, errback=None): + def __init__( + self, + link_extractor=None, + callback=None, + cb_kwargs=None, + follow=None, + process_links=None, + process_request=None, + errback=None, + ): self.link_extractor = link_extractor or _default_link_extractor self.callback = callback self.errback = errback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links or _identity self.process_request = process_request or _identity_process_request - self.process_request_argcount = None self.follow = follow if follow is not None else not callback def _compile(self, spider): @@ -53,22 +57,6 @@ class Rule: self.errback = _get_method(self.errback, spider) self.process_links = _get_method(self.process_links, spider) self.process_request = _get_method(self.process_request, spider) - self.process_request_argcount = len(get_func_args(self.process_request)) - if self.process_request_argcount == 1: - warnings.warn( - "Rule.process_request should accept two arguments " - "(request, response), accepting only one is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - - def _process_request(self, request, response): - """ - Wrapper around the request processing function to maintain backward - compatibility with functions that do not take a Response object - """ - args = [request] if self.process_request_argcount == 1 else [request, response] - return self.process_request(*args) class CrawlSpider(Spider): @@ -111,7 +99,7 @@ class CrawlSpider(Spider): for link in rule.process_links(links): seen.add(link) request = self._build_request(rule_index, link) - yield rule._process_request(request, response) + yield rule.process_request(request, response) def _callback(self, response): rule = self._rules[response.meta['rule']] diff --git a/tests/test_spider.py b/tests/test_spider.py index 78157a9b9..d23543f6a 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,8 +1,8 @@ import gzip import inspect -from unittest import mock import warnings from io import BytesIO +from unittest import mock from testfixtures import LogCapture from twisted.trial import unittest @@ -20,7 +20,6 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.linkextractors import LinkExtractor -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.test import get_crawler @@ -280,7 +279,7 @@ class CrawlSpiderTest(SpiderTest): response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) - def process_request_change_domain(request): + def process_request_change_domain(request, response): return request.replace(url=request.url.replace('.org', '.com')) class _CrawlSpider(self.spider_class): @@ -290,17 +289,14 @@ class CrawlSpiderTest(SpiderTest): Rule(LinkExtractor(), process_request=process_request_change_domain), ) - with warnings.catch_warnings(record=True) as cw: - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://example.com/somepage/item/12.html', - 'http://example.com/about.html', - 'http://example.com/nofollow.html']) - self.assertEqual(len(cw), 1) - self.assertEqual(cw[0].category, ScrapyDeprecationWarning) + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://example.com/somepage/item/12.html', + 'http://example.com/about.html', + 'http://example.com/nofollow.html']) def test_process_request_with_response(self): @@ -339,20 +335,17 @@ class CrawlSpiderTest(SpiderTest): Rule(LinkExtractor(), process_request='process_request_upper'), ) - def process_request_upper(self, request): + def process_request_upper(self, request, response): return request.replace(url=request.url.upper()) - with warnings.catch_warnings(record=True) as cw: - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) - self.assertEqual(len(cw), 1) - self.assertEqual(cw[0].category, ScrapyDeprecationWarning) + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', + 'http://EXAMPLE.ORG/ABOUT.HTML', + 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) def test_process_request_instance_method_with_response(self): From 9661a8dcfc79249e8e3e117eb70682ebc8e57a92 Mon Sep 17 00:00:00 2001 From: Sashreek Shankar <45600974+sashreek1@users.noreply.github.com> Date: Thu, 1 Oct 2020 06:46:12 +0530 Subject: [PATCH 20/80] removed datatype specification for *args & ** kwargs --- scrapy/crawler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 4c6b0e496..578016536 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -180,9 +180,9 @@ class CrawlerRunner: :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance, :class:`~scrapy.spiders.Spider` subclass or string - :param list args: arguments to initialize the spider + :param args: arguments to initialize the spider - :param dict kwargs: keyword arguments to initialize the spider + :param kwargs: keyword arguments to initialize the spider """ if isinstance(crawler_or_spidercls, Spider): raise ValueError( From 744f352d09a9ec519c84ec5243e62eef78c66f08 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 1 Oct 2020 14:52:23 -0300 Subject: [PATCH 21/80] Do not process cookies from headers --- scrapy/downloadermiddlewares/cookies.py | 41 ++++++---------------- tests/test_downloadermiddleware_cookies.py | 5 +++ 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 87f8152a4..d95ed3d38 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -97,35 +97,14 @@ class CookiesMiddleware: def _get_request_cookies(self, jar, request): """ - Extract cookies from a Request. Values from the `Request.cookies` attribute - take precedence over values from the `Cookie` request header. + Extract cookies from the Request.cookies attribute """ - def get_cookies_from_header(jar, request): - cookie_header = request.headers.get("Cookie") - if not cookie_header: - return [] - cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";")) - cookie_list_unicode = [] - for cookie_bytes in cookie_gen_bytes: - try: - cookie_unicode = cookie_bytes.decode("utf8") - except UnicodeDecodeError: - logger.warning("Non UTF-8 encoded cookie found in request %s: %s", - request, cookie_bytes) - cookie_unicode = cookie_bytes.decode("latin1", errors="replace") - cookie_list_unicode.append(cookie_unicode) - response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode}) - return jar.make_cookies(response, request) - - def get_cookies_from_attribute(jar, request): - if not request.cookies: - return [] - elif isinstance(request.cookies, dict): - cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) - else: - cookies = request.cookies - formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) - response = Response(request.url, headers={"Set-Cookie": formatted}) - return jar.make_cookies(response, request) - - return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request) + if not request.cookies: + return [] + elif isinstance(request.cookies, dict): + cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) + else: + cookies = request.cookies + formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) + response = Response(request.url, headers={"Set-Cookie": formatted}) + return jar.make_cookies(response, request) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index a3de307ee..aff8542e9 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -2,6 +2,8 @@ import logging from testfixtures import LogCapture from unittest import TestCase +import pytest + from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.exceptions import NotConfigured @@ -243,6 +245,7 @@ class CookiesMiddlewareTest(TestCase): self.assertIn('Cookie', request.headers) self.assertEqual(b'currencyCookie=USD', request.headers['Cookie']) + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty') mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) @@ -257,6 +260,7 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty') + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_header(self): # keep only cookies from 'Cookie' request header req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}) @@ -291,6 +295,7 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req3, self.spider) is None self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')}) From 44f0fde9057256dc16f24f22cfebc61626a94450 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 1 Oct 2020 23:21:09 +0500 Subject: [PATCH 22/80] Simplify TLS logging for the modern pyOpenSSL. (#4822) --- scrapy/core/downloader/tls.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index d9f3750d5..b5c6cc895 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -55,18 +55,11 @@ class ScrapyClientTLSOptions(ClientTLSOptions): set_tlsext_host_name(connection, self._hostnameBytes) elif where & SSL.SSL_CB_HANDSHAKE_DONE: if self.verbose_logging: - if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15 - if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0 - logger.debug('SSL connection to %s using protocol %s, cipher %s', - self._hostnameASCII, - connection.get_protocol_version_name(), - connection.get_cipher_name(), - ) - else: - logger.debug('SSL connection to %s using cipher %s', - self._hostnameASCII, - connection.get_cipher_name(), - ) + logger.debug('SSL connection to %s using protocol %s, cipher %s', + self._hostnameASCII, + connection.get_protocol_version_name(), + connection.get_cipher_name(), + ) server_cert = connection.get_peer_certificate() logger.debug('SSL connection certificate: issuer "%s", subject "%s"', x509name_to_string(server_cert.get_issuer()), From e42d82526a1a519ff1305b189acde064a40fbd8d Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 1 Oct 2020 23:22:17 +0500 Subject: [PATCH 23/80] Drop the conditional code for old Twisted (#4820) --- scrapy/core/downloader/tls.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index b5c6cc895..2b8990b75 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -5,7 +5,6 @@ from service_identity.exceptions import CertificateError from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError from twisted.internet.ssl import AcceptableCiphers -from scrapy import twisted_version from scrapy.utils.ssl import x509name_to_string, get_temp_key_info @@ -28,13 +27,6 @@ openssl_methods = { } -if twisted_version < (17, 0, 0): - from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name -else: - def set_tlsext_host_name(connection, hostNameBytes): - connection.set_tlsext_host_name(hostNameBytes) - - class ScrapyClientTLSOptions(ClientTLSOptions): """ SSL Client connection creator ignoring certificate verification errors @@ -52,7 +44,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions): def _identityVerifyingInfoCallback(self, connection, where, ret): if where & SSL.SSL_CB_HANDSHAKE_START: - set_tlsext_host_name(connection, self._hostnameBytes) + connection.set_tlsext_host_name(self._hostnameBytes) elif where & SSL.SSL_CB_HANDSHAKE_DONE: if self.verbose_logging: logger.debug('SSL connection to %s using protocol %s, cipher %s', From f47b120e2b67be7821a06ff77786c0bab2cfece8 Mon Sep 17 00:00:00 2001 From: Habeeb Shopeju Date: Thu, 1 Oct 2020 19:50:11 +0100 Subject: [PATCH 24/80] Documentation of link extractor usage (#4775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added description when using link extractor outside crawlspiders and created reference documentation for scrapy.link.Link class * Added link.rst to toctree * Corrected spelling errors, moved docs to Link doctstring to use autoclass * Moved link docs to link_extractors * Update docs/topics/link-extractors.rst Co-authored-by: Adrián Chaves * Update link.py Improvements to URL description * Update link.py * Update link.py Fixed line length Flake issue * Update link.py Fixed trailing whitespace Co-authored-by: Adrián Chaves --- docs/index.rst | 1 - docs/topics/link-extractors.rst | 21 ++++++++++++++++++--- scrapy/link.py | 17 ++++++++++++++++- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 11aa5c9be..da264fb34 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -78,7 +78,6 @@ Basic concepts topics/settings topics/exceptions - :doc:`topics/commands` Learn about the command-line tool used to manage your Scrapy project. diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index ed32411b0..e12ad45e0 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -10,12 +10,19 @@ The ``__init__`` method of :class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links ` returns a -list of matching :class:`scrapy.link.Link` objects from a +list of matching :class:`~scrapy.link.Link` objects from a :class:`~scrapy.http.Response` object. Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders -through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link -extractors in regular spiders. +through a set of :class:`~scrapy.spiders.Rule` objects. + +You can also use link extractors in regular spiders. For example, you can instantiate +:class:`LinkExtractor ` into a class +variable in your spider, and use it from your spider callbacks:: + + def parse(self, response): + for link in self.link_extractor.extract_links(response): + yield Request(link.url, callback=self.parse) .. _topics-link-extractors-ref: @@ -145,4 +152,12 @@ LxmlLinkExtractor .. automethod:: extract_links +Link +---- + +.. module:: scrapy.link + :synopsis: Link from link extractors + +.. autoclass:: Link + .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py diff --git a/scrapy/link.py b/scrapy/link.py index 684735f6e..e70667361 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -7,7 +7,22 @@ its documentation in: docs/topics/link-extractors.rst class Link: - """Link objects represent an extracted link by the LinkExtractor.""" + """Link objects represent an extracted link by the LinkExtractor. + + Using the anchor tag sample below to illustrate the parameters:: + + Dont follow this one + + :param url: the absolute url being linked to in the anchor tag. + From the sample, this is ``https://example.com/nofollow.html``. + + :param text: the text in the anchor tag. From the sample, this is ``Dont follow this one``. + + :param fragment: the part of the url after the hash symbol. From the sample, this is ``foo``. + + :param nofollow: an indication of the presence or absence of a nofollow value in the ``rel`` attribute + of the anchor tag. + """ __slots__ = ['url', 'text', 'fragment', 'nofollow'] From 159e2b2e2fbbcbb2b083a79e83e37cf4e60ee5ee Mon Sep 17 00:00:00 2001 From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com> Date: Fri, 2 Oct 2020 00:23:08 +0530 Subject: [PATCH 25/80] allowing to run .pyw files (#4646) * allow .pyw in scrapy/commands/runspider.py * aesthetics * added tests for '.pyw' * created class for testing .pyw files * name=None parameter in get_log * small fix * .pyw tests for non-windows * used @skipIf for tests * two more tests skipped --- scrapy/commands/runspider.py | 2 +- tests/test_commands.py | 91 ++++++++++++++++++++++++++++-------- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index aedd8c2ce..b957c29fb 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -11,7 +11,7 @@ def _import_file(filepath): abspath = os.path.abspath(filepath) dirname, file = os.path.split(abspath) fname, fext = os.path.splitext(file) - if fext != '.py': + if fext not in ('.py', '.pyw'): raise ValueError(f"Not a Python source file: {abspath}") if dirname: sys.path = [dirname] + sys.path diff --git a/tests/test_commands.py b/tests/test_commands.py index 2899e5f24..3e54a0948 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -496,6 +496,8 @@ class MiscCommandsTest(CommandTest): class RunSpiderCommandTest(CommandTest): + spider_filename = 'myspider.py' + debug_log_spider = """ import scrapy @@ -507,11 +509,23 @@ class MySpider(scrapy.Spider): return [] """ + badspider = """ +import scrapy + +class BadSpider(scrapy.Spider): + name = "bad" + def start_requests(self): + raise Exception("oops!") + """ + @contextmanager - def _create_file(self, content, name): + def _create_file(self, content, name=None): tmpdir = self.mktemp() os.mkdir(tmpdir) - fname = abspath(join(tmpdir, name)) + if name: + fname = abspath(join(tmpdir, name)) + else: + fname = abspath(join(tmpdir, self.spider_filename)) with open(fname, 'w') as f: f.write(content) try: @@ -519,12 +533,12 @@ class MySpider(scrapy.Spider): finally: rmtree(tmpdir) - def runspider(self, code, name='myspider.py', args=()): + def runspider(self, code, name=None, args=()): with self._create_file(code, name) as fname: return self.proc('runspider', fname, *args) - def get_log(self, code, name='myspider.py', args=()): - p, stdout, stderr = self.runspider(code, name=name, args=args) + def get_log(self, code, name=None, args=()): + p, stdout, stderr = self.runspider(code, name, args=args) return stderr def test_runspider(self): @@ -556,7 +570,7 @@ class MySpider(scrapy.Spider): # which is intended, # but this should not be because of DNS lookup error # assumption: localhost will resolve in all cases (true?) - log = self.get_log(""" + dnscache_spider = """ import scrapy class MySpider(scrapy.Spider): @@ -565,23 +579,20 @@ class MySpider(scrapy.Spider): def parse(self, response): return {'test': 'value'} -""", - args=('-s', 'DNSCACHE_ENABLED=False')) - print(log) +""" + log = self.get_log(dnscache_spider, args=('-s', 'DNSCACHE_ENABLED=False')) self.assertNotIn("DNSLookupError", log) self.assertIn("INFO: Spider opened", log) def test_runspider_log_short_names(self): log1 = self.get_log(self.debug_log_spider, args=('-s', 'LOG_SHORT_NAMES=1')) - print(log1) self.assertIn("[myspider] DEBUG: It Works!", log1) self.assertIn("[scrapy]", log1) self.assertNotIn("[scrapy.core.engine]", log1) log2 = self.get_log(self.debug_log_spider, args=('-s', 'LOG_SHORT_NAMES=0')) - print(log2) self.assertIn("[myspider] DEBUG: It Works!", log2) self.assertNotIn("[scrapy]", log2) self.assertIn("[scrapy.core.engine]", log2) @@ -599,15 +610,7 @@ class MySpider(scrapy.Spider): self.assertIn('Unable to load', log) def test_start_requests_errors(self): - log = self.get_log(""" -import scrapy - -class BadSpider(scrapy.Spider): - name = "bad" - def start_requests(self): - raise Exception("oops!") - """, name="badspider.py") - print(log) + log = self.get_log(self.badspider, name='badspider.py') self.assertIn("start_requests", log) self.assertIn("badspider.py", log) @@ -696,6 +699,54 @@ class MySpider(scrapy.Spider): self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) +class WindowsRunSpiderCommandTest(RunSpiderCommandTest): + + spider_filename = 'myspider.pyw' + + def setUp(self): + super(WindowsRunSpiderCommandTest, self).setUp() + + def test_start_requests_errors(self): + log = self.get_log(self.badspider, name='badspider.pyw') + self.assertIn("start_requests", log) + self.assertIn("badspider.pyw", log) + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_run_good_spider(self): + super().test_run_good_spider() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider(self): + super().test_runspider() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider_dnscache_disabled(self): + super().test_runspider_dnscache_disabled() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider_log_level(self): + super().test_runspider_log_level() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider_log_short_names(self): + super().test_runspider_log_short_names() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider_no_spider_found(self): + super().test_runspider_no_spider_found() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_output(self): + super().test_output() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_overwrite_output(self): + super().test_overwrite_output() + + def test_runspider_unable_to_load(self): + raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") + + class BenchCommandTest(CommandTest): def test_run(self): From 797a6690c07ae90f7a2d703832e2fe44466d656f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 1 Oct 2020 23:11:11 -0300 Subject: [PATCH 26/80] Tests: use classes instead of paths in settings (#4817) --- tests/test_dupefilters.py | 10 +-- tests/test_logformatter.py | 4 +- tests/test_pipelines.py | 2 +- tests/test_request_attribute_binding.py | 14 ++-- tests/test_request_cb_kwargs.py | 4 +- tests/test_spidermiddleware_output_chain.py | 92 ++++++++++----------- 6 files changed, 63 insertions(+), 63 deletions(-) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 95a4fca0d..680bb6dc8 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -43,7 +43,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_crawler_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -51,14 +51,14 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_settings_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'} + 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'from_settings') def test_df_direct_scheduler(self): - settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'} + settings = {'DUPEFILTER_CLASS': DirectDupeFilter} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, 'n/a') @@ -162,7 +162,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(SimpleSpider, settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) @@ -191,7 +191,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(SimpleSpider, settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index dc5be398f..6381f895b 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -193,7 +193,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): self.base_settings = { 'LOG_LEVEL': 'DEBUG', 'ITEM_PIPELINES': { - __name__ + '.DropSomeItemsPipeline': 300, + DropSomeItemsPipeline: 300, }, } @@ -212,7 +212,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): @defer.inlineCallbacks def test_skip_messages(self): settings = self.base_settings.copy() - settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter' + settings['LOG_FORMATTER'] = SkipMessagesLogFormatter crawler = CrawlerRunner(settings).create_crawler(ItemSpider) with LogCapture() as lc: yield crawler.crawl(mockserver=self.mockserver) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index c72f1a338..ff3af9a74 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -68,7 +68,7 @@ class PipelineTestCase(unittest.TestCase): def _create_crawler(self, pipeline_class): settings = { - 'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1}, + 'ITEM_PIPELINES': {pipeline_class: 1}, } crawler = get_crawler(ItemSpider, settings) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 907117468..00c532c41 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -92,7 +92,7 @@ class CrawlTestCase(TestCase): url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".RaiseExceptionRequestMiddleware": 590, + RaiseExceptionRequestMiddleware: 590, }, }) crawler = runner.create_crawler(SingleRequestSpider) @@ -119,7 +119,7 @@ class CrawlTestCase(TestCase): url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".ProcessResponseMiddleware": 595, + ProcessResponseMiddleware: 595, } }) crawler = runner.create_crawler(SingleRequestSpider) @@ -149,8 +149,8 @@ class CrawlTestCase(TestCase): url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".RaiseExceptionRequestMiddleware": 590, - __name__ + ".CatchExceptionOverrideRequestMiddleware": 595, + RaiseExceptionRequestMiddleware: 590, + CatchExceptionOverrideRequestMiddleware: 595, }, }) crawler = runner.create_crawler(SingleRequestSpider) @@ -170,8 +170,8 @@ class CrawlTestCase(TestCase): url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".RaiseExceptionRequestMiddleware": 590, - __name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595, + RaiseExceptionRequestMiddleware: 590, + CatchExceptionDoNotOverrideRequestMiddleware: 595, }, }) crawler = runner.create_crawler(SingleRequestSpider) @@ -188,7 +188,7 @@ class CrawlTestCase(TestCase): """ runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".AlternativeCallbacksMiddleware": 595, + AlternativeCallbacksMiddleware: 595, } }) crawler = runner.create_crawler(AlternativeCallbacksSpider) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index bd49179aa..145a4e9b2 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -50,10 +50,10 @@ class KeywordArgumentsSpider(MockServerSpider): name = 'kwargs' custom_settings = { 'DOWNLOADER_MIDDLEWARES': { - __name__ + '.InjectArgumentsDownloaderMiddleware': 750, + InjectArgumentsDownloaderMiddleware: 750, }, 'SPIDER_MIDDLEWARES': { - __name__ + '.InjectArgumentsSpiderMiddleware': 750, + InjectArgumentsSpiderMiddleware: 750, }, } diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 2f454addc..029bf8bd6 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -16,11 +16,20 @@ class LogExceptionMiddleware: # ================================================================================ # (0) recover from an exception on a spider callback +class RecoveryMiddleware: + def process_spider_exception(self, response, exception, spider): + spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) + return [ + {'from': 'process_spider_exception'}, + Request(response.url, meta={'dont_fail': True}, dont_filter=True), + ] + + class RecoverySpider(Spider): name = 'RecoverySpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.RecoveryMiddleware': 10, + RecoveryMiddleware: 10, }, } @@ -34,15 +43,6 @@ class RecoverySpider(Spider): raise TabError() -class RecoveryMiddleware: - def process_spider_exception(self, response, exception, spider): - spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) - return [ - {'from': 'process_spider_exception'}, - Request(response.url, meta={'dont_fail': True}, dont_filter=True), - ] - - # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method class FailProcessSpiderInputMiddleware: @@ -56,9 +56,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): custom_settings = { 'SPIDER_MIDDLEWARES': { # spider - __name__ + '.LogExceptionMiddleware': 10, - __name__ + '.FailProcessSpiderInputMiddleware': 8, - __name__ + '.LogExceptionMiddleware': 6, + FailProcessSpiderInputMiddleware: 8, + LogExceptionMiddleware: 6, # engine } } @@ -87,7 +86,7 @@ class GeneratorCallbackSpider(Spider): name = 'GeneratorCallbackSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 10, + LogExceptionMiddleware: 10, }, } @@ -106,7 +105,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider) name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 100000, + LogExceptionMiddleware: 100000, }, } @@ -117,7 +116,7 @@ class NotGeneratorCallbackSpider(Spider): name = 'NotGeneratorCallbackSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 10, + LogExceptionMiddleware: 10, }, } @@ -134,32 +133,13 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS name = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 100000, + LogExceptionMiddleware: 100000, }, } # ================================================================================ # (4) exceptions from a middleware process_spider_output method (generator) -class GeneratorOutputChainSpider(Spider): - name = 'GeneratorOutputChainSpider' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - __name__ + '.GeneratorFailMiddleware': 10, - __name__ + '.GeneratorDoNothingAfterFailureMiddleware': 8, - __name__ + '.GeneratorRecoverMiddleware': 5, - __name__ + '.GeneratorDoNothingAfterRecoveryMiddleware': 3, - }, - } - - def start_requests(self): - yield Request(self.mockserver.url('/status?n=200')) - - def parse(self, response): - yield {'processed': ['parse-first-item']} - yield {'processed': ['parse-second-item']} - - class _GeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): for r in result: @@ -205,26 +185,28 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): pass -# ================================================================================ -# (5) exceptions from a middleware process_spider_output method (not generator) -class NotGeneratorOutputChainSpider(Spider): - name = 'NotGeneratorOutputChainSpider' +class GeneratorOutputChainSpider(Spider): + name = 'GeneratorOutputChainSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.NotGeneratorFailMiddleware': 10, - __name__ + '.NotGeneratorDoNothingAfterFailureMiddleware': 8, - __name__ + '.NotGeneratorRecoverMiddleware': 5, - __name__ + '.NotGeneratorDoNothingAfterRecoveryMiddleware': 3, + GeneratorFailMiddleware: 10, + GeneratorDoNothingAfterFailureMiddleware: 8, + GeneratorRecoverMiddleware: 5, + GeneratorDoNothingAfterRecoveryMiddleware: 3, }, } def start_requests(self): - return [Request(self.mockserver.url('/status?n=200'))] + yield Request(self.mockserver.url('/status?n=200')) def parse(self, response): - return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}] + yield {'processed': ['parse-first-item']} + yield {'processed': ['parse-second-item']} +# ================================================================================ +# (5) exceptions from a middleware process_spider_output method (not generator) + class _NotGeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): out = [] @@ -276,6 +258,24 @@ class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddlew pass +class NotGeneratorOutputChainSpider(Spider): + name = 'NotGeneratorOutputChainSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + NotGeneratorFailMiddleware: 10, + NotGeneratorDoNothingAfterFailureMiddleware: 8, + NotGeneratorRecoverMiddleware: 5, + NotGeneratorDoNothingAfterRecoveryMiddleware: 3, + }, + } + + def start_requests(self): + return [Request(self.mockserver.url('/status?n=200'))] + + def parse(self, response): + return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}] + + # ================================================================================ class TestSpiderMiddleware(TestCase): @classmethod From 004b40a7193a7c0c5138abe764cad6d1d9a4fc57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 3 Oct 2020 00:53:55 +0200 Subject: [PATCH 27/80] =?UTF-8?q?as=20soon=20as=20=E2=86=92=20as=20long=20?= =?UTF-8?q?as=20(#4825)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 675f55c38..4d2580a6c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -140,7 +140,7 @@ original pull request author hasn't had time to address them. In this case consider picking up this pull request: open a new pull request with all commits from the original pull request, as well as additional changes to address the raised issues. Doing so helps a lot; it is -not considered rude as soon as the original author is acknowledged by keeping +not considered rude as long as the original author is acknowledged by keeping his/her commits. You can pull an existing pull request to a local branch From 892dd9da57c5cf005670743c8abfdfffc7027acc Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 21:00:58 +0100 Subject: [PATCH 28/80] Adding support for zstd in HttpCompressionMiddleware --- scrapy/downloadermiddlewares/httpcompression.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 727c41466..b1abb8b5c 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -14,6 +14,12 @@ try: except ImportError: pass +try: + import zstd + ACCEPTED_ENCODINGS.append(b'zstd') +except ImportError: + pass + class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be @@ -67,4 +73,6 @@ class HttpCompressionMiddleware: body = zlib.decompress(body, -15) if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: body = brotli.decompress(body) + if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS: + body = zstd.decompress(body) return body From c6c3f2ce661ef9296302b9d489d5a2cb2e49f481 Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 21:10:40 +0100 Subject: [PATCH 29/80] Updating the doc entry for the HTTP compress downloader middleware on zstd --- docs/topics/downloader-middleware.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 06e614941..9645ed5fd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -684,11 +684,14 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ responses, - provided `brotlipy`_ is installed. + This middleware also supports decoding `brotli-compressed`_ as well as + `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is + installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://pypi.org/project/brotlipy/ +.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt +.. _zstd: https://pypi.org/project/zstd/ HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From da3171d4f71d50a15b08b76b2b1e06bddfa56694 Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 23:18:58 +0100 Subject: [PATCH 30/80] Using the `zstandard` package than `zstd` for supporting frames both with and without the content size info See also: https://github.com/sergey-dryabzhinsky/python-zstd/issues/53 --- docs/topics/downloader-middleware.rst | 4 ++-- scrapy/downloadermiddlewares/httpcompression.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 9645ed5fd..7c63a623d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -685,13 +685,13 @@ HttpCompressionMiddleware sent/received from web sites. This middleware also supports decoding `brotli-compressed`_ as well as - `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is + `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://pypi.org/project/brotlipy/ .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt -.. _zstd: https://pypi.org/project/zstd/ +.. _zstandard: https://pypi.org/project/zstandard/ HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index b1abb8b5c..56421a6ba 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,4 +1,5 @@ import zlib +import io from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse @@ -15,7 +16,7 @@ except ImportError: pass try: - import zstd + import zstandard ACCEPTED_ENCODINGS.append(b'zstd') except ImportError: pass @@ -74,5 +75,8 @@ class HttpCompressionMiddleware: if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: body = brotli.decompress(body) if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS: - body = zstd.decompress(body) + # Using its streaming API since its simple API could handle only cases + # where there is content size data embedded in the frame + reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body)) + body = reader.read() return body From 50e1f35d1fcaeb2e0a0c4fb29894cb7c686a33cd Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 23:43:12 +0100 Subject: [PATCH 31/80] Adding test cases for the zstd content encoding --- tests/requirements-py3.txt | 3 +- .../html-zstd-static-content-size.bin | Bin 0 -> 8066 bytes .../html-zstd-static-no-content-size.bin | Bin 0 -> 8063 bytes .../html-zstd-streaming-no-content-size.bin | Bin 0 -> 8047 bytes ...st_downloadermiddleware_httpcompression.py | 26 ++++++++++++++++++ 5 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/sample_data/compressed/html-zstd-static-content-size.bin create mode 100644 tests/sample_data/compressed/html-zstd-static-no-content-size.bin create mode 100644 tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2247ed917..2eed2f5da 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -16,6 +16,7 @@ uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython -brotlipy +brotlipy # optional for HTTP compress downloader middleware tests +zstandard # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tests/sample_data/compressed/html-zstd-static-content-size.bin b/tests/sample_data/compressed/html-zstd-static-content-size.bin new file mode 100644 index 0000000000000000000000000000000000000000..b5c2038e893481e2835ac74bc790e3718986e426 GIT binary patch literal 8066 zcmV-|AAR5`wJ-goRU`obmHPmitfU$&;EqO3lFJAV16>@^C8(hY1ZsEGeDXVhj51gD z{wbwTMW0fDVIjsC8zUwJLIPd^Tmj940%8FN4lqZfU`(TE6bp>O3tV-7e>YaR^FV=)2Knk9h* zPD2Bxu`I2bEJ}oPC|O!VVVHKB!!*Tl8rtL0DmWw>I4vT=w8gSua2St>L(&)r1jK1r zB93OkEFwfW4Gr3a353%c%i_@@6d(g3YGzZS2 zoTW`Hi$=w=8$L-PHPy)01wDyvS31MEJ`~NhjNz41g9-ZJB>s+5DPeP;2ej=V*+6! z2AsyCEXzc~31Ax}lf`M!Fs)Hq1*RRwGC2*?lnG^m!iYdlH$+BRYv$Pg*Bn!uBmIj$fL>Nb+ zI3N}nq$wB$#lixsfDwsMEIcR{9u$&CQHqNO3cLt#DP1=;z9HvN0E*SB0B*i|V@? zo!0dgG3lazM?J5RBBIe@s@IB`{`D4B*O(MBjS}6Wt}`j>9cd;_G)lU*(nS1E>Agxb zjnrOTx<i{{ zuB!54rdz~ydTv6L_*KKIR4?Lcu5WP{s>|c5E+#Mj_g#pa>{ZT1y12Y}n{th}_Fg9A z1<pTU+@2?hd|hVob(Mx!|A9L6+?2gL$o znujl=Vp?bP`Io;tI|2zX28DP;Bu<1Tv0&gpczD1Z49GyhvY;TT5Mhqv;Q{e@YIOW5 zqWax`XR5Q;t%z4Y--^b(_2;8+t;tDU>hpQ0m`+OFBE|bpS6Zlk`l0deZdK^3BEqA> zRR2pCDs;Ng>RwcLUCNkuG%jD}5BD@tF&)x8I_jk&p7V)G(`gZ{6xZdtwNm_#i_Tl2 zqB?K&qnD;qh`TiXT=d>C?`4E=@0jSv^u2Ns{X||@`%Yx~m|nN3bgg-Nx_tMh zb2;^SUoY!f553Y~t6QVm_vRBZ^WMz=)_7=tdGooqRzH&-Za(+c=|tQ_UuU{hqXP@< z0cji$4~m6_Csd@4D6L2Ibahm`g!G*LBSHufrT@5&`E=gt9~CLC;}Jc7L|=%~6FWqh z)KQ{@tS;?4N|cF5C{yuIl!%f;qC}+9QL>bGMwkhqyiClabV{lyoz%Zjs#H$uh%&8( zGE+h*FQsBA^U|pZX{r@dh?4q3dQWF3rYIBo$(bpW`jX0&^eGck=d==2g+9fQD#cPN zrs*jaN-9yMJ4&e7Vo6!qdOH>KdRm#P*!oJDnd^{Z>vwA1tum=Ey;6m#)M8huSXwL< zyYE+pnzz`xLRIF)^nSQ}mAF#nVoIqle<}8*>gtp&UCI#3`$D>wuEmgpUg~u*WtaXF zV@FG=YEsvw?77xb#?qIJe2AxK>MKiD`*%9?=$NK5X1y|B^~%4xGPYPE{XHF$5%uVv zh$RPg#J-Nu4u_8E88LN0hKV6`NVMXQA;#D(2W5Cv4reS8VvI4C3^^pk7z44y7-P{K z3Za>!Q7F?WBrt{yiq@z^(6q(i%pd>(0YCr&0Mioy02mAmh>Sx5DrwgN6o3I56b29! zEDRI~0t7&C;P4u`C=>t&LSZl%C>~Ta1x^BJ0QDms#I{8t+xDvw^)H%OmM^UtD6u=U zJi2g;L;Z>qIeYC%x77AvCUM0~2dZsvqsQWjcw9QI^3oQ^-Ny=pBn5wXbuZyGOE-C_ zU+(88f@qO{Z!@13Ld9B{bEg_PV51mS_}EE@qw|Xtd&KjUfOP~dg4w^u)KKYBv)CAw zot38EujidM6)6_Q0emIE5v2Dd^J_S#jAd|478Ri)!r?XqW)yq}yGjGyFZpbke@%9A z_^u{F`#3YR{645{L-~!AtK$yGb6vk!62X2+S8;$e6dz#?0lkbchOLSGA;3MZNDi7_ zDZ`<`7%Pq~Ov@c1`J>Q+#XEwZ=En{ju0tq)SgwM`F;(vui(^FjAxj@&3Tt+_cON8; zz{D;>Q?B)}jJ^@EIH)ri__uIYjQ$Y@q47ge6H$0uQ(bqJX;zw;fYKs74qQlWnM$zn zt42$TaFlT17RDKQ`q8l82lUt`FP*I7d0CAn3p5-cre}g%ko2+aBjS01-TT!%Qrs`2 zL~I4ZXS$tk!Q*FY&%tSqr}IP3GN6x2d0K&-ZUNiTB0Ivy7Z|@j7m;$}>sJ?+$a1=&4s!@7FS6X}iQx6KvgDT?Bz{7oilbL9z-4ct_rAsPzI!Y^_3E}j__m*@@3lVL2ujan z-u(=F9dJ)WDyvlfO1+zK;~Mmg=xq%#`vU6msw@-M!y$M-#>{el>f_pp|83onyRevw z@EiId#6snzKtlyy;h30OWXlx=z~ z8}Cu#3gc># zIsI;^khN5tfp}$v>?3qU-7kD0p8a@sT=af|0#_@H>GQR&&9K?&p^xx*zgWVa9Vz4P z4;Zh}^^L+z-surWhn#H`tOz}3p7HvZ9Kw{D2R?l=7)&}Gyqjjo#btC!7Syv>l%_4S zLwFzbs!lxJBv+9Xhv{~*+wa~&-|>*CZ`3;@nD5bjTf>=$P3rs}kA#>Wz7~BbuZJv8 zndrY)4{tW7znl}>9`ep3Kh#XT?jiGkI#$4%dBoD`P5%KpD0O2=r;|Mb<&a&A2?8>sh%T{-p_2KWE-H%9O1BV(V`{m17l>qV~z=?BrOh;_ok7@rd2|~ z`IqZ~j#yB*?T5UQcoG{c>rM7YkW2>8;YPtzq;4M3q*%5%@@zBzElOkE`!xUm@V@_h zF-3O)mbFwQ7h|5XRm0Ma9d?>`gzZKP8)duDY^I3gof79a|Lels+-KtuK4VlhC3Ih0 zS@X39#s>(w_aSJpT9w=&J}k{$ads~zVPbbbhdHgGLH>|G%lp-{0F#l;doyZWe3rMJ*W&^`p7joRO!v022)e(S+p z#yehuj3s;f^${>C5j_r4M@v^RvCZ_?fQoR~@4}WtcGlRGBi|pxdi$syZ}exD$01+G z!M>kq1o{kDtgmOKHpPtot=pS5O4zuF=()lO#Q8LVERrYO{T8<%5qO!rCdcTd9Diy& zC!{vY%$8?enEN)b92Nl>-FXKbAROmvSiFp3vPTh4Zp%8Z=gHJ?K>4LnS(xX zb1J2eqR#`H`qoU?o?WSpX&zX!Zl6h?>1f9m)UeGc;iMW6RBN` zkl5>1k(4{f2WZS$^!@ zVpweBLB^Ep_DMvN#Cg|2ESpjZP&ZnkE5Kw{5h08s;?v_4zyDJ|M$~D7c0cAczQA*4 z7-2EajlyM1j2A#K9ukMp75cR6ap^JDiQXLOfGnP;iKT?t%&rN zW6;7Lzin2>bV9bWHd_5g$q_v)YP1k_D3ry?S7nHbVHw+=Vu%rQXzPTumK-5KS(A;& zI_u;DQK;-3XAyWu3o+}Y=g>xm!&@972xxn$eg8vt_ardqgyqTVVZNKWmp~shY`27o zq69i#6~P3nHkYnM3*&Hgg-N^nKa0yH`ZIH~mG|}O!Cn<_vP~wR2{kxG{GrA*RvY8K zNliJ&ZTjR9^!=>yVcRHcZv-zA@5v#vp zYI;Fe6~Qj#_X>8S-WlaX9v?%^mbl-kD|H4URprK77}T9wsfA$5#tsiB@w$k-t2hKB z;ckC-UZ+?VQq@0oR}SiGfG>2g(5`^|$T}f|n$P~Ac@h%RoBOO)QL0U-2!P%wips{d zsLED_O>Q@N_{5ZgmBp$Ed)<`1hO#8uDj?_B5&nnuv)yVNfeq``N6E(e$>H=yi3F5z zzy_?9I!OZLvNVpiytoK=mlJ$ulD1hq-oO!53_>++*WX4Vi1}7lS2-&}5bnJ|t~%?P zMTEb0rtS#zI2RWRHDwG}h3{y=&8>Qr>j)!3u*(hn3>26YL%s@o1my&q359knG`dqz zyk30)IO4i98PPUCphScEb9w~$piA#FaT3dUWv)o-pKBElLM2yeoAZMUpHI8uN>O^v zQO}LjuKd*pH2f1}na}!3u<@8A)Sw#kF4XQ89j;t)w(`+N4DWmn?d z2+i9@u~Dq`ZcPfB=IjZG?h=BI+}}+Lkz+7!EQf@RHL1(nSRA_-oF9*2U@q;UN@u$VVQ8KO9Bvr|F6nB#ZEtf8?B^8gnu^v z`WG&m zKEQJZfNxH;;~fY175Z-j&m#Hdqmg+R-GHQ&gD9o0HrWqwurD#R-ySmGv-=HI^c1gD zwYgO}Sti&zRVeh`-zlbaJxPTod6hmnHB5!=sR#GjjqI&I4d)OoY{jCiG%^9l`AFErDd9JHjyE;%@t zoSwtaPOn#wVgJ$`LE{=$fGi zEM+B+x6g!hLK!By3B{`z+qL$?*CYWGQ)=tpGL@Ll5^qV$>QEAfhw)z*nPM$S6o7Ih zPB}WUZ@V^N5TKb_H){!DoiWxV@iZfO7$#tj3H9|I10G4Lhw9o@v5pZbX^G0yBDE>E z|M1D4{8|9ZO#=&)kYbd&S{&&EXvJ17;={#0B+H$nB)%W7rnMddqR5PS43SNe;E;F9 zE+QMlb_G*0toS}$mRuj?y+v7epym)z7Z*RiJ!kDUyGx!0q#goF z1lU@}erQBm5)WJ5R6+dPE02?nDoLSL6FPT->dG~d1yVNUjUUQs69|>6qQVy4P6z8egty%z#uULZ&s2jW?Fp4iQKq^Wz~HyVK$F)P4V7DLN8Y9!+F`+s znMjga|7GOl_+X&4Yx4(;8>S(-{XLklu7{62fB4o9#sA7TODSq|iYeuGFw-i$ALJwN z?YCcejmWGw_gsJF8AQ>ZjFHOfvhE@DzBc6g;L7?@tUm03-g@0*e(pV7BHJZZakVL5 z3tSnbG5*j7S#h&Gcvoe0R%iolUJz&v@cCRHj!~e_TnU>%gmH^RUWYh7Vn#D;m~?tb zuU91Uf>ZwMkbrR1IxN3DxJybgk`Q>`iCW05&3=^63x1eOF14G3enY>+)Go5O#604MLGWJRHEUjnhpxgKg>cKb(@=Fvn|J%$(wPu(&hPAF+r8GnOx ztum~If0)stw2OxFuNI4bk5~l2m}^0?mriM`sW;69uY9h=Vb(Cf0*U2sy4{>uAw%B zmEQ2}+14&Cu-OiA%TFJYs#r*A z)qynN(DPUDsJJ9+gOoC+Aw|wc5S6{q50eQ*p^r`s1kvFnwbU|(kkUi$93rM%zaD9d zKM;gFhj z^O4=x(ZU?v!;*H1)LPX`2>=bGQEkTmJFYLS`d8S(-k}egdju$|jTk*FF*uV6U_@|@ zF67`#tjJ{MOP91@OW2{?20f)Y;;NuYLPh#Mog7UaG>(aKx2S@$}hMGhrsiA+qxmf%?3EFI|SAoj}a zXG$P0BuNp$3bkPn{;4;MdRR7kMEh=P=&L&u9D0*M-p~(25WEBby}_n^<(CAU57?P`b1-VsB6%%MEM$O4iO*V8pw;o%2mG0`0C^h>T2oiL&; z4R&bCN1-EepFzb5&1BdoF~aR!R0X6}B043_OI3o-I+&2q(r7uR4r0D>!=QmuFB_dc z#Cd;!C_GL`*pmI&4>q!5tu`kwjfe|zTvjZtTG1)=g;$!8TSusve#Lmrp?xxO6q0;2 ztc4FD_D7#paqN}5hyVE8I#)3PpWNT+=kcq#uruz8Pg&StbtUaE1Dv%t&A)%2qlZS?rpybhE#F9kknmP+7Fn%e(p)Bikzjs zcR?+z>z0)1fs*;ryQ|0mNP;L{`(hO2G*75pX;+M>D%f45d<|;iFVkU)W7BDdCoHDZMKHGZ}fDahIn9VA30Ww&x=UM=SKb$wa>@yLXOClaN6gNTHP z9DQv@Q1xLW;JQ&`4&{bmhJ4BgRg(**gUYT=D?zte@jkjMMa3q3vpvVv^M8SRBpUl2 zLpemb6)#?PfMRYM6i*kHj+U?%`$pV0?7{x{_$IY_oK@u zkRPiRQarhDiz4%x$WRVtGT*2`$(^|mp!j-<>F~%-hSAHL!e7lv79~ymqYD$oGd}~T z=R=M&7xyD#K)|>|OF%;A~C=c3?=8w2fzC4ql1WI<>&7| zi6DkQ?l1Z<9A<3{Ihn03Q1PPFrcD%B04SP1R7AY5;9mXqSsB}?!;SRcQB;W1l1+49 zCkOOzJ=c@$Gv`Yyw%v9W&-~bM6XTwaveGIflSPj82XY4+q$E4hO4k>PuzU$I5*T=* z0xBms9%iY1P9R$;^u2JzEj2V#T(%A=Tq*u^&kdP^-hMCmgHpsS%<+F6QqbD_d<^+2 zq&&gVmRy3$wWQod1t5diDF+o(>iFdl+UU@Vu{EE6gwipxWqEJS-4zp>)t+s z+CmgEpIf%5cVC5QEM7S;&F zpNMBaX`fV?!T{488eyRFVNiT%K0z2oY|(O+v~@Z_7nW}3Nx zmCsSsxh)?|OR~R|-qkyEAi>n?Lp$x2wsrgVwa~IJ2yx zg)qj0`!t6(&At=G4uAmQ0mNC-E+nFiNfqr0baw3p?4DvAIAwv&#u3OmDi(ND^by8T zEVKXBZi%IJWhNr%lMy*J+w zbV-M7DA?|fpWc_gx{Y5?yTCy_hgBu0ex(2j`f+xKbP1SEm7n*Pa1r{74DruGNsbVL zTw49`*DnkEFstbn-5>JUWa{LCm0L@QeZ1cOrfa?3T5F%eR~bs5QGO*Wckg~uFtwW| z`Cm6sdO>HzPn}}t7n9^DRpfn%JJn|rX!jD&(yLv~rmxBWEZ6n(3m>aN+TM0m-*t+9}@VeBC4QW82|tP literal 0 HcmV?d00001 diff --git a/tests/sample_data/compressed/html-zstd-static-no-content-size.bin b/tests/sample_data/compressed/html-zstd-static-no-content-size.bin new file mode 100644 index 0000000000000000000000000000000000000000..3d494192e2c72294810309eb14e8d55e0431fd1c GIT binary patch literal 8063 zcmV-_AAsN}wJ-eyIF~>I5=%-jHWmwButy&FfHONt+7lRiP9buC_okxG);TpGzz4dXqx1-M#8ky6bI)x zG!o@Bi=#bSV_BRAaV(Yzlf{Gykw(I_$)bVN9t(&9Iv#{dt=WU^pFYb;7T5QlP>$ONY?N;{22IS>mtaNrz=#bW|tA_kns zqAbfq!U z0Z>pT8WEhvqA(&7OFPZdnk6D}9F0=pFj~_n5XrJ&nzOVPaU=`JX_f|=NJJP%qBtNH z7^Eo}1;xSwtAG)SP%Jzs79JFmM^l!CgCl{&LV@FuSS)90k4KAGK(O{`ltyvR0%I(i zL$Safkj9}{U=SieAaEWq00Be+3pfBk5lC?}eXVd+jp*mfb+R!RpI3#OcZ=$~8lBel z6*1|ee@8v9ks_keVXD`PnEv$^Ro9plF^v-4qOLP3>K$n&O*Bfnw$eoWPwBl%GmX?< zT)IZaU7G1Ty}P|Ds&mn)dJ(@)Uf&P>t9*~i20*$Q%YQ4C$9CUiHr&rS8s1Z2=Nw=>mM!q3-u5#y01^w{Vn2J7d_#fl&-4s zVy0Wfb$V_>l=xNET+FY!#f5A2gvd#ca?g9Y{^6=zlrGb)N_)|ZU*|e^U00zla?+po zbn%`BmUF!3Brp)MJ`>)Lxrpm$UOgrLN2{wm zO=#WYU42p4$HQmpdVOX7w|RFf)0e0B+f>}kREczbr|ZAXg_vHqsdTM*d%Ar0rgJ&< zd0#K;Y*U z4-blkg(p;`jwr21^mKJpyoB_e{v$#N5vBjQj`?)n=^qs-uHz9se?(u1(i1yGnAB0C zgsd*@J4%#^M<`SAPn3v~L!v~a(owRMcSe{Ap}b7YqjXBDD4o>5P^wf;>WDI}g)&n@ zC@-aADD%>(2x+PnQ;3rKLV8bUC#EP9`pKCollqd%l=LYRQs=Z1Q-waokSfJeDyHcv z6-p{mr8`Qf*kVap*?Kz_^LkpDs@VEUnVIX5V(WKm-K{dIFTGNQsnlXusaRSp6}#_O zg_^h6x5n_xnmJB&0#25py#291I915YC zqfsc+C?qh342ssMM9{Rw;LIQZ0Rcb&0RYnz000;a42X20~#l7$_c8GzCrqXaMyi9mKXpA=~z=5%n*cSe7rX87Q$kvpl+R zi$nd26FGbBNw?JYU?y?JOb4oMZ==WJiFjN(t@6?q$KA&YgCqricy%w~G)p&ms9)~q zCxU2^e{VCN7DB~ZnRBNaIbfq0RruIRhoke06nn(;lz?>vErQv<#?(;hQM1?>m7SHQ z-mmAKHWeur#Q}UJz!9YPB=c)Hr;KHAOcoWPBEsP|1ZEU`2fIoG-7ooUn14-narmw# zLHjr}v;01&ZA1Bul&j+o$8%l3SQ5d0Nmp@zG!!3U4FSE3Fovy({2{*kTT@+km1$O*n1IqEJPuq)ZJA22@vBBl zig1*0;1?7iNf!+JnJW|{*qeN^4 z!e_djZo%VcYR|!GqdrGaSYM@6!tPi5=8+jQ0?X=S!<6n8>IG%IfoSVP)+iIyE}0OV zL#OjY&N85nN_kp=oNfWz(IPv-#upgBJ{OU4;_F=dwc&mburUsT9QqbC6?cSFjTzyv z%CB#Y|A}=Rs%mBg{+PlsvEy#xZkh2XM$evI462Lh$eyse1hw{~e0&T4wBd3kckv%C zS1zz9`v1S-@u}ZK{&Cu8R}Knq^WMH6f#C}R8wG)g%2(+@%U}(KxTyUy+*Z=FS4lk= zb?**ykLamaRPWa^VQIU>Q4?(4T3rNzZWo~)OT8Y7EjOc9*G5Z4Q@v1Bn*25uqKgP^*e(K}eiT`cgkh`#$itroy zAjCrDr9eXkUg4Oqx;okXoBE|%iMbMM{Xlr=qn1jw@3eagE5kwb^M54o3%Ll*<#YtN zQMdjR!bA{fw^~iqGWMq={2^f4#lrPLVzY^`^JKv9L2ZT+WsEZ%B7kGCc)wV}o*gOU?hhER z(e;hOOy21cMu(hj6s!n6W}fl-mmI>BnFl_7G8jxc9K4%m$i-!JNfy+zSCpnLvqN|v z^r}ug-6U6$6o=_{vfJ<8Lf`R_sc+OfBbe{eeOtqshfV7I9*=~W9=;ZRD6fYsPnqby zR}XJCr@x#N+aB`HBR|wkyzU|Me>zscnt8<1=}rFuIw*BxNvD%N0_BihiwOxw>&*HM z(|6#X19f8)w5gsWklxR1qhuSP%N*gbaM7YA>jPtCzGIFFrX(#6llP{P$EH<6!11Pl(WF?mIPz>W|1C;m-TO5E|M0&5doe|K z0hYB?Bo||zvQ@*nx%VMxv09bfAU-V0h+DrSXw@`38f+0-wfmhLvJu{9Rtu&PwHMsiQX(k{YV279 zGpilv6LfwEfi`e0J?vc}8KF?I7RAL9KfC&T*fjHh~*Yi&y_`gmbPb|{tW6rmU=6Bgs$z#H;T6iR2At_ zOt1@hhWMG?uW6nNn>@z$NBw&Ai0HY(2*mj`fh>|I-2E1}AQ5<(ye7x!r5t~1JSU_! z%FLE$U6}heuN)Qu7~OdX93UL$YFNCCVX{XNPHxLOuII_-Ij!pcDDe??PpG?5vGr}a z%6Q#2oPJ2($TE!))I$4I?lq@zMSP}6m1APoLPoUThI>>n6L|`=G3`4QESZBoaC0i9 zkEDBnbw<10xh*3#HD!-bL%TL((x-A7w=~G~UHL{yAqT%ZDMfkK;kXs1qB!tvQn_B4 zr2?47!_e{p$TPm30^E)#6F1mK9qv?GexK_G)^N&SX%D_MT*-C|g5 z<3Yxh>-I@RlEit}K`fh62~am$p)0^-RuLhLBI48I6uv8EZ)rsC5=zuJ)V)vC%bTO(_5x!GNZymYP+*p{5sW>zK zVlCJxp~bz*xO4QZmCBM}3W3}ik~XGxW=O```o=|_z#wTcC8_#{8?#HF?xr&SvEk80 zTVo0$=TBE>qP$>PFp7f|V|OvNA4*?w3ZwpKv|QG$2#le z0#T^!9A^=DM+-6Qr038^hQnJNAqZ%DseS)LcK0MO=Y-|S>S4Z{xtBm6G;FtoiJ}BL zUKPOvt2UReL<{3^b%ja0`#+1zCHgaSvX%Gs>A_wVZ?a7$p9wWMMEs$~HC7wrzDZ3v z$8GxL5%m46@nPF2Yi|T!uSBzCIb}}pczx1KX;rC?S?=o!U}p*cv+Mfs$`Px-VQP9o zR~5l7|sVj8`B30$aS{T%wTB(I#%Ek^4C-J(7ysJ0_BjIj; zcV4Gh7E;webyp7RYJe|vu+Xl6{Kz^XgPPC&pm`D!(wqCNRZ*%y@s+R+A1LD*b)AR^|Re-8-We$)kn$3`pMz+Mu`NJaKHwv zl{!fRX6SB39r!Og9DlEHt`PP`qA! z0XX8iGa1n~K%hi}`g3{&_@GPgGjS5jd1bCh>Yr;B4?-nZX`Azd3!hKB;!06^%~8*d z)2{s02Q>T>WSP(UNwD#lB-Ef9^DflxS7f`H$92v5wUj76}mTWU}44aeCJg;)(?w-VvzbVf_cHc;TP0= z-DQuwZO-H0s9IT>2WvXLvBL-?wfjGwyZQRg#v2|vD=l6I*Mec(=YjEjD^G(WXWJXsY%N?mRv+Rz@1)8wM*y$+$(~W3^WqRmT>E?Wo@H0!+z8Fv zMzK+>^=?fHn&#{Yi0%@Cj@;i(3z1_mZY+m{jWwyu+gKdC7n~oDVPG!p%&FlqN0zlX zTE0O7qCQzpgKsz&nxK!~>gsl)iO5GDgg+cb?d3k`DF3g@JH<{vfE%r(MTCDg{`wa# zn&gn)C{ho9aaLfgiir1%4sl})YeO<~m)>Ah3D!ajhY7EUD0Wg}+7cYS4>a4n-T1}TKErjhGuA99;^K<_PeTRz@H2cmY`irh{ik-&%4x)V(1 z55C>CaRsXJkdvIK#fx%bnjOguFEg5?=THCmgHWsW&Y!+|3*Y1 zKs<1y2K_jFm@Kd(XDS&!mvbSSWgfwICDeJd`HXm_mh%b?R4+8snjEyG#x6NHmzx(UUr7~8e>!`CDM6H{vI-ZGV#&Ju4)%IZ)OhKKQA7nx!$NECo_Bu+Ux zv2VLJU=X00S~qJ6VVyD7B=IyOc^D>OjtTYk9RnUos)y>@Rk4l{DQSty(;~GgxBu|T zp8Q$>%S{6dlaOMRx>_9R18BuoE#kw)ha?~U`@y`F0G(MneLdzXhm`;)|8pkT0BgaTFy+AO$r zc|>z;ZOXA6HfQ{FKzXfD?}K%i8aao}#w5#~qa?l`ucoyg0;0%_c?^+FlHibc$}S=s z!*&HzF|7DLT$l;$S7m6}l!Nl;-H8uNeo~!HMX@7Xhczr;^(6^xc2{RCG?s^_9n6yJ zrx$FhmFiv|kG(}%cA(}EP!|_JzCCB{Hsu&U?6}s1J$$f1IRGJo{JTq@1*9GVN(9(i z#(ro-S`rUi-BdyR+bfTgjVei@RTDaQg6hgOkp)sV<&7W8X%h&Qs-nUc-Q`CQQ8HD` z{^C$fc1{QDJcPI1BgPcLG0#+kBkc*5N>Qe|7{K7S#XytS7!8$MY)9Uv9NJ;QjhRT2 zTK{F_ay-3^u9La`ryj?QLH}ffZlrDV}9;ETq4^gRdKZ`Ukh9r zq%r=`23c{lJa|`SbyjEtZC(&)4eyUtO)H*D`Jh)3rF_I8?-;0;u&*kv(G3bz~Vx01q&(qS)Y5(p>+xsJ08CjMw zj96&m3T@9*Isi5zobu@pzkOFpkTK}tChPK9PY`x-`%jdq)M69fH}ByLdR$Yu9@Q=h z+u?sUMS+N?L;xr6qhv*)ZC?Vh%DEnDUUvIOljhMxRXv6n2~XWJAxd3TLBshLF-j?i?bfT)!S^ia!v9 zb$N?J+4zS_k3~knp*_PTrzkI@xN2Fgdk6yrUP{tZv@y)$V<9LDydiD|pc|yB!t;^c z*U`cp-NTZ0iPT!vO9=oCq)~0g|2wWPt@>Bk!rq||ntKE&s*M;uEHOBf31CEUjV|Qi zORUIb=1Z5fU`ya_?aZ}1#Abkf&@814#@Mn1!Tsf&W**r(9XhI9JvgO5bf1?|kKISBp67iH8wnXi zGBziPrB~%RSJ}N_K zE+k12!3wou5dNt*i+Wf#dPMtfYUryw6C8SzLEg|0LlDA~+&gK6$U@fuv1B(i6WP#! zbu0RQS@)pN+9kI=?ColY%ia+~eaxXez{moU5ZBW+S>fRaWiinl?DR{n6P+-kEe&>P z%15Ciai2lO3C(2KCo#h9TvP?5RU$ei%}Z5+&N`To(9&o*rVe7hal@d2QZE~wKE!!{ zfG9jpNZ6A7*bg?cVy!kOFO7%`aa>j`tyF4pQxv(?ticeYCV09(!Faw@v$ngXdK$~E}R~O#- z=sZ(Z}-Z^5+F4B2@(Yve207!x;Ui)Gca%cnTUB*QwczeKQ+r#6y7Ld`OIPXfsg3;yw2fcK-zCXgSi z6;eF8Z;K-HnaEHMWisEWK*^oC51{yZis|skPKMFTo5EksNfsqd{G$sK#WOzxr{_bC zGZ*(GVnD#SLrXwIW@(*}E4JNs70>+Ga1-O6jA#V(Ms1Bim-eMF%lSfq5>)> zI38xHeNG@-DfGQ?#4R;6Q(U$VDO@T3bk7Z$g5G{F_=8f!EX?tL9a7NR`+N-fDx^HY z(Ux3-%C)Kp&Y8un&v|F3^NC_#JkvU8u$0fLK4OW;1Qa_W!MMaSxLLSdZ|mMZg4#k9 zGM`(vsCeouYh`#%KB$zT&0vbp;x2;M4MJA(;U*~3np@aJ1-s-{fPUy6K}3pzLe|67 z;}D^W4%}L;ENmB3|F2R`y2p@GU`VDWvV>}7WpHM~ink?_Ga`9u8YPGIa~9SJ#Gi;~ zKWU#-nZf|m92#Mu@?lVXXg)z0T&wb-A0J=?DH(CQ?3n!D#j|%!#}r^Bs)|iw4GZ?q zC4JPB6q*!Of!zHjxUfYtC8C`N(1x2%HdhVy)@;#om9%v_Kp1s)8xK){`evHBfR)ct z)a3pOMMWetiqN0)^p(rHABV!>05>p)B#T)LJSWuVsl5pJ21UKd-$O4-Os3!@(=Z$a zdtysai~}M!jCy7L_ts(qY1wrm((ihE8CMER*1I!vlbb*G8Mmv+vV+#N?>MupqJ=QV zgZngxHqE{h#SVY~-~q&0(k>*Tj7b&k33PVt1?-+;95`ix&BhVPIw}@;RP+(XP%N|! zH(RO|3em0IZ)KNlmMD}mJPF1r+sPXP#1qSi1*lgj*6NX7rpoAmp-G3$BE2`?5Ohh0 zY$(|7j-TF_zPgQHPrJZDJcm^!sD7mY3Hot%hI9#-O_iVbmT(dJiVX43LP?Ggf?Qht z@YgR3`!K8N7Tq86*ktPDf|XlKh<&`?{-$fa-CAp(!dDqepHY4#D|hdHQZTifCi!1C zP&}Os3O%?TsHs! literal 0 HcmV?d00001 diff --git a/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin b/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin new file mode 100644 index 0000000000000000000000000000000000000000..97bdbcae01d8838c3ee046f227b91b99893daabe GIT binary patch literal 8047 zcmV-#ACTZEwJ-eySQYyK`m2ZriV$moT92z_% zjb-wPIGDz0$^#6}5>XnZX%w0S38$gK!b~Pkn^+oiz+qbRU~nFb37FO_2_$eD8ZeDz zY0YF&BAi3X(i#fGw9_1>DUQ?79*>mIZ^uctjkM#yB7#PQwy$Gz(@C zA;M{B&?ZbEoYq(tj~1Z-X%GxfV~Id`s6euKaF_$(;lXJft&tq1MJyte#?l(5J#d!E z#93O?D3YZ$iy~<(%Azb7h%=cut?_V3NSKHN6O5%T7F0M5jb*}ElnACda2Dk(ZDLt8 z62-x3OJg*}At7Pf1czx6XK9UP(nyr{m_Pxtkf3SW1E*0S%|z2Ar!^9$ou)WA$DxrZ zr&%2B(HhI*G>BudOqeVtOo%iRrcD+Noc35i6bcrQWpSFqIH#cv&`=%|k_7`iAd|=f zrm-}KM3Zn@!#D_RW%p$t8Q`OT0J3h(xcq-Uao(*Di@{8bgR-{^y1gK&Ry44sEeHR=RIBgruQ~; zeYfz7e5(-Ei@2KWTik`}^0=yt$&3Ge7vd&+m2;6UE-&7uT;r|1m&tem^s6*^jf=)r zxpec^*1h~DU#>#9ZyprqaLE{$)+|kefqgQY0n796X^sO~Hoj#v;is_`(EmFMybftyrrym;c?pB4qDk3~8RR2pCDs;Ng>RwcL zUCNkuG%jD}5BD@tF&)x8I_f3jIiHv`ofgqbab2!kE5-k~=)4sws`FMqdTAX|w3+w@D91jnQg@q$h zM@5v@BYL_zDqcc*PX7@hgox6AT*rJm@AQv~6xZ>Ho+Mv`>uF`GV(TkqX0Ahut>39rt4!)kuT)_wwb)fEmR4pSyYE+pnzz`xLgvNv zez<&Xa>A$`H!?Lb{f&#gKzu>UA+?m;MuDM@y+{QrD&Ixzd`$BOAhLYeI2144jt1o zV(Nel6GP~bXvH5xjImn|%J8Th&R8PE7-K9Ma!80V24aaZ#-ceCLNiCBP^M8xU<{c6 zW6+A!s8sN@wcyMk0099&0098g6951h3=D`&Ljo#o*Z~xP0U8tr5ELv76bJ$YKycs) z6R{{100u%~Fc2snR5S%n0%!pBBOPGdQ;==LD-nJdO=c4|$1C}PGP^V5qHDG|4zGZh zGuoa6mpUIw5*KB$2&)LTzs6=^=~3KbQ!DjV8nIu?cXB|a zEQ>?>3gr>1_ax;Pa}Z4|B4ke5@^yrrZ7847!w!y0bF*K2HhgkThvLXtO-T1~R3KwN zbg&IoG}3s-AjgxqUydGO{3V6r7}qdy1gMGYM(7M15&82CbX-9nG({<+pM%j~9B-KB zA7SrN;DeP%1k~n7H*D=gSo*N+gJzzpi!at8g7+ae9)X{_cQ_w^IFG>6FS=Gv|FD4H z2%82T2t z8JQJ_s^P=o*v64g>*6U`ja}~PhuZ|aQkJ+aFY76dfyNx#&Wm+|0$f9}2I-MkP)~oys1m zZxWugw@wlJtRZz@ARO<|WdeFQF7GFjSvE?2eLK0|rX6zg7Jd;6hdxcQ0Db9dsGvI> znO4`eHtVHcs8)$QOD!Lcg+A_5*%xT}p90Hp4FCL1l6P=64=nrA)U%VV z=35zSR1(z?&~mYeUqH09BMhF5`1d@UVW*675)Kx<>wE-4H=PRQ%8Zo55)>|r=qPO5 zZ!q$P)-SW0(EgqO9o&WE6+eK1aw4vN!!X`69@m1ag<|@DLj{_pwuSjCAjl)=ME$l= zghyPx5P4q&`~-2rDnW`;sTL9bf{HxswJU~3eZ3QVJITL<#^G^0yZ76#&}T8^ z@s09-1l9L&Z|gSmL=;87hag+x`<~f9H0dGRQ{KhzyTTjh>EGwXnjQ{&WY`a9YU0Tc z8UE>IJc8OIj7}f*N5MhC8;d+0{s_oJKJj8*I1YG=)0uu%_Z%8D*3_odj*xIaM>ncP zU%mE-(ZVGjEmh6oJF69W3bD2Z&uw!$SjI`v-uAI8q3xbxBmyV{f`q%s5QQ#nm;HbzfAg$!m>{kE}TSu(#Njl|&IAz+i}5 z#3Q)Z6gV2@BLKB~)D2S+UO?8((^$<5Ue!|Al-xG95Ch|{9XSxPeKDUOIbH%8TlR+XBT!c&`VInXOG;y6 z^7LA95V5o0ST4Wstbvpx-5>4u=36@y(Vs1k<15DjydRgRmTTC3e-pRleQMPx;@>dEj8t^N06a? zn6c1PIgT497foJ?Hc6#`-%lxpoV7b{QB$`b@W)}yI0{Tr3t%Z8hL%u3o^kC2OZz_g zxXC=~aHnGW{dB2z7A|2bL15Dzzs#Jc0vxyTWpeFW!Xwb$DB95?X7~4q1?|M)H01u( zAGh>|Yz2zBey`i8T$$UgBvCW2=W{zx^44%_0pNzVqg&IW2oF-xwhrNL-gB^wi6k?uVqM*+)n-^FpCWm#n+ixUor|2$&oZXb z%&?Aiv5kxGf!Vr%O;T(Qw**L@>!t$yvES(0t1%govsYKwKzWhaaGwUv2c22ayt#WB^7zJ+Wu*(e`!v*W)9WQKId zO@&B)?Y ziMqTP~yYlOs^Uvj}Me0h;2%5hSPqtgOM3LCjsLEdgs5W=3#Jh12jbfxd^PeROauGXo z_Li6M>0wfpaa=@9i!3!lQ2v448cPG?9#T`Bf3yDN(S1MY@zHFQuN$F!R~BP#Ws#ir zcv|TdpsLhYQSOBbV2qXJ*@gR{+Y#K~U@<*%Rh8;4AbUl(QKLTOgE>Bq(3V8ssaSPJ z@T$stEmhXt-cy9ZjN%UGCsDh2Vpq8YBi3#Ys$b`^tZS-&S1U(=PpCGNR%ik>3}>1$ zL$?0@a6bvw(<9s4IgpuKK%U&Fj4mK8we z*v9Y&@Yy1?&9@DU>_;PF{rPabjS3wP8o~xS;+jODf*^LAoNY^l`^pK=Y>#cak5};# zc7jmOZ5IGSNdA#;rTQvGBLsWze{+Re&)|px%1KJV$&eTcU$Uq6I>QCzt9D=Sw&$)V# z$(5FWUg+s)@l2JR(A$6=95B9OoZvCNM%Oc}3|AKE!;+B+GM=ycgrRqwKhz*N=Ed9Y zR+QV!IJ&7BVBYem@Q3BU!m4fF8JCoYL%bpu?rOj&t%wOk%lbA*wD%~7Xq3oui12;$ z%H50GU~xrw+_$OH5G#=O1r8$_{uW}Teq>y^d&R-P6`&bBv>{aQA3oOt&2 z;z^m+jzE6%>z-jbXAU8fu=dw9Fi)?9fe}J&8#ANi*1I_=*q-z8Uv!sN9o6}6K_Wi` z;;~#UEytv;{YJsqH8|fW!@zIa_gjPi91&@uxBL$PLPR9VG04GO?u?m4y{=WMCc+5h z0WZVRZLcJQ4*&eBsZ*r$1KbFmXhb+?+*+VH4-|mObq4|op;axBTMqWcr)*@Q1?w_A&GblrN-tsp{*>izh`P0zL#Smnq?lr zc_rj|v*C?6rE0hQbneK3z8N<*@#oV8#Xned#K@uxTEbkwWsULZ%1T>7qwG1 zQc0DOA6j>sVs+PwHIxS#%+U;%1xpxSIPrenvvWOwWB>Nbp0xb@A zi!i~G#eTT%km92cJ{VLbpct0aOyG(Q3fKQqaSfQq$M#6TjO3#Z_R`kpg#=pK+O8yu z*u;8`_Arg{NgO_X-VbI(ls8tTUs~F*qDX|Jb|AzLtE4<|Hs$FEmw*f+oBKopr_9>cMdI6Taqa+Z+|v0Z^QhA+Plnq~q&ptiL( z<*mu_?uSE`pR{^Y#o5utA)V!OwEgxrv8|+XRZGDypy@cDd8TxJ(7Vzkn5!8aLfq4f@WHU`iFe%y#4eGy%U)==bi|zJVO-mkumT| zT=qX~|7)aN9~{LW#czjr=&f=+W~T<1Q6l>#Rk5`xXH`xa%$e&j16i?H9+<0M$RlXO z&_!=87O3Nq9$oS>qhW7F+(-Q2m`%lJHdS7MsG}^=TcwAFmgV$ZM5U05SJ8V61i2C{$@>xd0sE4jHllVZs z;34hodJDxMFH!Y5wpluqaXb5~Fvu~>`%RF}_714cS(RiNe#b_k_#$tTg~H;u8|_2C ztpibDP&eH@(IE)yYD|lRY1wU;&TMvSxaBAhy9q4Bw(2vze0cn^K4>nv_8`R=Y1E#x zA`qoe`9t>uA@ZX;0znfvNr?OzLlb-OZ4S|r>*Gc`><=(&UC#d@y#68XvE+z8+}LoY zDN@KNaZ*d8Jxr&8mr7d7Z;ZP7SeD5G{)!tK6oYhCc!sh2Kv|gEa`;>^a^U<1hhAU?T#DB7>uC0x%1%zCwok5|#0rd`V49lM+~F&3aLCxIzm8t@VN_rx|A z)M9PkAMe_lHgafkZ=O*-Gh}O(RlzecPnB&}+%?8}DB@%K^o+v7G=va%5?6>OAxbd8 z;&e78YyTw!WOLrwp$yCn36X1*`iL-tFk7HZ>;q(buf0sxAIf>D1e+)Y=C*~xu!bBe z3`IZ|1czaTvo-a*EiG=)kH|dV4kAu_EFP_7p2}+wE@YU>*tH~JykQ@P0F1}6cfg3P3*7-C$!_RmvZDd&R92Sv`hDcK15#j#=QFzmQVN(&MADm#th&Oj# z8sR9!VJFzwyJFM#MXogXx9*~thKe>cC)H#sDI^GJSTY|M#g9&?;xjJk9{BIQb+2MN zKDjm1|M5ZP(AH8FowTss>q^_o3~+g0>X9n1cCDjIXxe{yMS z(Bno0WHAuVMI>PJsCb*-y6(5zhH2-MGv))>KI(KXA2E^r#o2+-kB*LF&pBJp%8)Ma zgVqEKES{3(s(Y$8;U=%B!16tl{b?AcO`dybs$%(4PhC)1qPR=?q8YyV(Sa)l{BjWO z>ApBBDa{ivSJ@R4s|tG-310)6hn8s}0oruht1IH5JAB((6a z)JP#89c#tL4yUl(-gmCv#PR{OA+uQsZUoo16|GjH^Wh;3_E(^k+>PCzIFeMEN=9#7bCZR)inDx1xjv-;l|1r};lj9tjTp238If+)5cQH^DPvOxzmP%HI;R zO++NsOj;2q`Hh0B&3>^be@j0)AVqC;1R5=fAfhI@gbFhbl4mr66ye$hoD|TIaIUZl z@a`rf9?03+uLo-6Ig4bFsaezClMv-62LFy00Q~65Nan|`FcjbJ+oDL^Oly`S8O3f? zu*`Pu0w{VdiZOX4t;5LLO{Z4pC3BS~?xUNDBAFk+sevK4lZ!h!F#u@XaZEravw({x zfr`nl1}~J!*<_p$EwLD2D25XA<$+%P|ItH0u>$kgpG1)2AFo$^_!?$R4E@Yj5~>(e zs?#P4Jb)LPu9PCOYr#4F?pYZx)bU1o-cdw}(#a-@yVFGUXFbTOYR-rVBA$a&7B z2ufK$VZ8K4e8C0*V|)f^mT`c=@>OM{AxRA#EWFnXW5YRWwJIu`s+$Xi%Nd zMqrA|<6eqa3_@1;FjEvIlUvxM0#qOLAA{xpb1@!oQX(g6rnR`X->_& zAIFWu;cj3INrtr8e@-YTsYems4T^D*e~VtM8&JVX7Q>(k@Wdsds73c~R(fU6Zy$^O zwq>_SWccvFGR_J1y?1BmCO3ZU8E&_nWmkNo9p#L!ilV|u)_122ttVV36k-6_hlgQ~ z#$A3fGCGZPH{RJV1uWf=@sr6;m>BU-)@Cf&R54^2X*p@*(rjNVC@ifV-IDFnY&DSM zPW6mg-^uF^l&6m22C%n~o7JPi<=NCR!1Khpss=hTD#`$)reM9NkF+URcs3JH;VtoJr literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a806f55ce..63a69f7af 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -20,6 +20,12 @@ FORMAT = { 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), 'br': ('html-br.bin', 'br'), + # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin + 'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'), + # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin + 'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), + # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin + 'zstd-streaming-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), } @@ -80,6 +86,26 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b" Date: Mon, 5 Oct 2020 23:55:48 +0100 Subject: [PATCH 32/80] Minor adjustment to the test case in tests/test_downloadermiddleware_httpcompression.py --- tests/test_downloadermiddleware_httpcompression.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 63a69f7af..7e9f9cc5d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -101,7 +101,8 @@ class HttpCompressionTest(TestCase): newresponse = self.mw.process_response(request, response, self.spider) if raw_content is None: raw_content = newresponse.body - assert raw_content == newresponse.body + else: + assert raw_content == newresponse.body assert newresponse is not response assert newresponse.body.startswith(b" Date: Tue, 6 Oct 2020 10:50:17 -0300 Subject: [PATCH 33/80] Docs: mention limitation about Cookie header --- docs/topics/downloader-middleware.rst | 5 +++++ docs/topics/request-response.rst | 12 ++++++++++++ docs/topics/settings.rst | 5 +++++ 3 files changed, 22 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 06e614941..ae84b54fb 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -207,6 +207,11 @@ CookiesMiddleware a warning. Refer to :ref:`topics-logging-advanced-customization` to customize the logging behaviour. + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + The following settings can be used to configure the cookie middleware: * :setting:`COOKIES_ENABLED` diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 30b1945d0..f3aaa2c8f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -61,6 +61,12 @@ Request objects :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If ``None`` is passed as value, the HTTP header will not be sent at all. + + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + :type headers: dict :param cookies: the request cookies. These can be sent in two forms. @@ -102,6 +108,12 @@ Request objects ) For more info see :ref:`cookies-mw`. + + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + :type cookies: dict or list :param encoding: the encoding of this request (defaults to ``'utf-8'``). diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 06234c5d9..71331c841 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -352,6 +352,11 @@ Default:: The default headers used for Scrapy HTTP Requests. They're populated in the :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`. +.. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + .. setting:: DEPTH_LIMIT DEPTH_LIMIT From 9b1f86b613d2039b0a66ba2b527a7e9bffadaf3a Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 6 Oct 2020 18:50:55 +0500 Subject: [PATCH 34/80] Use f-strings --- scrapy/utils/iterators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index e140e3f6f..3b504e56a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -35,7 +35,7 @@ def xmliter(obj, nodename): namespaces = {} if header_end: for tagname in reversed(re.findall(END_TAG_RE, header_end)): - tag = re.search(r'<\s*%s.*?xmlns[:=][^>]*>' % tagname, text[:header_end_idx[1]], re.S) + tag = re.search(fr'<\s*{tagname}.*?xmlns[:=][^>]*>', text[:header_end_idx[1]], re.S) if tag: namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group())) @@ -45,7 +45,7 @@ def xmliter(obj, nodename): document_header + match.group().replace( nodename, - '%s %s' % (nodename, ' '.join(namespaces.values())), + f'{nodename} {" ".join(namespaces.values())}', 1 ) + header_end @@ -56,7 +56,7 @@ def xmliter(obj, nodename): def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): from lxml import etree reader = _StreamReader(obj) - tag = f'{{{namespace}}}{nodename}'if namespace else nodename + tag = f'{{{namespace}}}{nodename}' if namespace else nodename iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename) for _, node in iterable: From e40788153ca7dee9e0d4f8ac16a9e17278f79058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 6 Oct 2020 19:13:29 +0200 Subject: [PATCH 35/80] Do not consider about: URLs invalid --- scrapy/http/request/__init__.py | 6 +++++- tests/test_http_request.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index ef58deacc..498f1b052 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -65,7 +65,11 @@ class Request(object_ref): s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) - if ('://' not in self._url) and (not self._url.startswith('data:')): + if ( + '://' not in self._url + and not self._url.startswith('about:') + and not self._url.startswith('data:') + ): raise ValueError(f'Missing scheme in request url: {self._url}') url = property(_get_url, obsolete_setter(_set_url, 'url')) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 0a303dbe2..74579dfc4 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -43,6 +43,15 @@ class RequestTest(unittest.TestCase): assert r.headers is not headers self.assertEqual(r.headers[b"caca"], b"coco") + def test_url_scheme(self): + # This test passes by not raising any (ValueError) exception + self.request_class('http://example.org') + self.request_class('https://example.org') + self.request_class('s3://example.org') + self.request_class('ftp://example.org') + self.request_class('about:config') + self.request_class('data:,Hello%2C%20World!') + def test_url_no_scheme(self): self.assertRaises(ValueError, self.request_class, 'foo') self.assertRaises(ValueError, self.request_class, '/foo/') From 2e734e6b35f686f62dea11e9484c7646c7c49ca8 Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Tue, 6 Oct 2020 19:51:05 +0100 Subject: [PATCH 36/80] Minor update on the import order in scrapy/downloadermiddlewares/httpcompression.py --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 56421a6ba..f504302e2 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,5 +1,5 @@ -import zlib import io +import zlib from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse From 156bb0a1d413c7c6acafc30003ef98030a06e47f Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Tue, 6 Oct 2020 19:53:40 +0100 Subject: [PATCH 37/80] Fixing the minor typo on test file path in tests/test_downloadermiddleware_httpcompression.py --- tests/test_downloadermiddleware_httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 7e9f9cc5d..4c5bfc577 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -25,7 +25,7 @@ FORMAT = { # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin 'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin - 'zstd-streaming-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), + 'zstd-streaming-no-content-size': ('html-zstd-streaming-no-content-size.bin', 'zstd'), } From 1a597d5e3dda7a467d69cff51df7e731f2f2d6b5 Mon Sep 17 00:00:00 2001 From: OfirD1 Date: Tue, 6 Oct 2020 21:54:42 +0300 Subject: [PATCH 38/80] moved the sentence about processing pending requests when a spider is closed onto a generic note. --- docs/topics/extensions.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 14096ada4..519f18b63 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -257,6 +257,12 @@ settings: * :setting:`CLOSESPIDER_PAGECOUNT` * :setting:`CLOSESPIDER_ERRORCOUNT` +.. note:: + + When a certain closing condition is met, requests which are + currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` + requests) are still processed. + .. setting:: CLOSESPIDER_TIMEOUT CLOSESPIDER_TIMEOUT @@ -279,8 +285,6 @@ Default: ``0`` An integer which specifies a number of items. If the spider scrapes more than that amount and those items are passed by the item pipeline, the spider will be closed with the reason ``closespider_itemcount``. -Requests which are currently in the downloader queue (up to -:setting:`CONCURRENT_REQUESTS` requests) are still processed. If zero (or non set), spiders won't be closed by number of passed items. .. setting:: CLOSESPIDER_PAGECOUNT From 13ae17aecc632dc03af1bd8b2b63e48301147cc2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 8 Oct 2020 14:04:52 -0300 Subject: [PATCH 39/80] Add xfail_strict=true to pytest.ini --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index ca8191f42..1c95f715a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,5 @@ [pytest] +xfail_strict = true usefixtures = chdir python_files=test_*.py __init__.py python_classes= @@ -40,4 +41,3 @@ flake8-ignore = scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 tests/test_loader.py E741 - From b55c911ddc21a41a6e4204aaf239a16e892de7b5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 21 Sep 2020 10:59:55 -0300 Subject: [PATCH 40/80] Fix CachingHostnameResolver --- scrapy/resolver.py | 57 +++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f191deac6..0350c82b9 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,4 +1,5 @@ from twisted.internet import defer +from twisted.internet._resolver import HostResolution from twisted.internet.base import ThreadedResolver from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple from zope.interface.declarations import implementer, provider @@ -50,6 +51,27 @@ class CachingThreadedResolver(ThreadedResolver): return result +@provider(IResolutionReceiver) +class _CachingResolutionReceiver: + def __init__(self, resolutionReceiver, hostName): + self.resolutionReceiver = resolutionReceiver + self.hostName = hostName + self.addresses = [] + + def resolutionBegan(self, resolution): + self.resolutionReceiver.resolutionBegan(resolution) + self.resolution = resolution + + def addressResolved(self, address): + self.resolutionReceiver.addressResolved(address) + self.addresses.append(address) + + def resolutionComplete(self): + self.resolutionReceiver.resolutionComplete() + if self.addresses: + dnscache[self.hostName] = self.addresses + + @implementer(IHostnameResolver) class CachingHostnameResolver: """ @@ -73,33 +95,22 @@ class CachingHostnameResolver: def install_on_reactor(self): self.reactor.installNameResolver(self) - def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, - addressTypes=None, transportSemantics='TCP'): - - @provider(IResolutionReceiver) - class CachingResolutionReceiver(resolutionReceiver): - - def resolutionBegan(self, resolution): - super().resolutionBegan(resolution) - self.resolution = resolution - self.resolved = False - - def addressResolved(self, address): - super().addressResolved(address) - self.resolved = True - - def resolutionComplete(self): - super().resolutionComplete() - if self.resolved: - dnscache[hostName] = self.resolution - + def resolveHostName( + self, resolutionReceiver, hostName, portNumber=0, addressTypes=None, transportSemantics="TCP" + ): try: - return dnscache[hostName] + addresses = dnscache[hostName] except KeyError: return self.original_resolver.resolveHostName( - CachingResolutionReceiver(), + _CachingResolutionReceiver(resolutionReceiver, hostName), hostName, portNumber, addressTypes, - transportSemantics + transportSemantics, ) + else: + resolutionReceiver.resolutionBegan(HostResolution(hostName)) + for addr in addresses: + resolutionReceiver.addressResolved(addr) + resolutionReceiver.resolutionComplete() + return resolutionReceiver From 8fe5876597825a4df03bd15e6b1cd1682b806de8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 30 Sep 2020 14:56:17 -0300 Subject: [PATCH 41/80] HostResolution implementation --- scrapy/resolver.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 0350c82b9..0bef555a6 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,7 +1,6 @@ from twisted.internet import defer -from twisted.internet._resolver import HostResolution from twisted.internet.base import ThreadedResolver -from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple +from twisted.internet.interfaces import IHostResolution, IHostnameResolver, IResolutionReceiver, IResolverSimple from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache @@ -51,6 +50,15 @@ class CachingThreadedResolver(ThreadedResolver): return result +@implementer(IHostResolution) +class HostResolution: + def __init__(self, name): + self.name = name + + def cancel(self): + raise NotImplementedError() + + @provider(IResolutionReceiver) class _CachingResolutionReceiver: def __init__(self, resolutionReceiver, hostName): From 868826b346714ceff821886536a3b259072f8396 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 2 Oct 2020 15:16:58 -0300 Subject: [PATCH 42/80] CachingHostnameResolver tests --- .../alternative_name_resolver.py | 15 ---------- .../caching_hostname_resolver.py | 30 +++++++++++++++++++ .../caching_hostname_resolver_ipv6.py | 19 ++++++++++++ tests/CrawlerProcess/default_name_resolver.py | 11 +++++-- tests/test_crawler.py | 20 +++++++++---- 5 files changed, 72 insertions(+), 23 deletions(-) delete mode 100644 tests/CrawlerProcess/alternative_name_resolver.py create mode 100644 tests/CrawlerProcess/caching_hostname_resolver.py create mode 100644 tests/CrawlerProcess/caching_hostname_resolver_ipv6.py diff --git a/tests/CrawlerProcess/alternative_name_resolver.py b/tests/CrawlerProcess/alternative_name_resolver.py deleted file mode 100644 index 2c466da04..000000000 --- a/tests/CrawlerProcess/alternative_name_resolver.py +++ /dev/null @@ -1,15 +0,0 @@ -import scrapy -from scrapy.crawler import CrawlerProcess - - -class IPv6Spider(scrapy.Spider): - name = "ipv6_spider" - start_urls = ["http://[::1]"] - - -process = CrawlerProcess(settings={ - "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", -}) -process.crawl(IPv6Spider) -process.start() diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py new file mode 100644 index 000000000..f9eab3543 --- /dev/null +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -0,0 +1,30 @@ +import sys + +import scrapy +from scrapy.crawler import CrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution) + """ + name = "caching_hostname_resolver_spider" + + def start_requests(self): + yield scrapy.Request(self.url) + + def parse(self, response): + for _ in range(10): + yield scrapy.Request(response.url, dont_filter=True, callback=self.ignore_response) + + def ignore_response(self, response): + self.logger.info(repr(response.ip_address)) + + +if __name__ == "__main__": + process = CrawlerProcess(settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + }) + process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) + process.start() diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py new file mode 100644 index 000000000..3340d2f84 --- /dev/null +++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py @@ -0,0 +1,19 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes without a twisted.internet.error.DNSLookupError exception + """ + name = "caching_hostname_resolver_spider" + start_urls = ["http://[::1]"] + + +if __name__ == "__main__": + process = CrawlerProcess(settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + }) + process.crawl(CachingHostnameResolverSpider) + process.start() diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py index 60d91b68b..05a98fbec 100644 --- a/tests/CrawlerProcess/default_name_resolver.py +++ b/tests/CrawlerProcess/default_name_resolver.py @@ -3,10 +3,15 @@ from scrapy.crawler import CrawlerProcess class IPv6Spider(scrapy.Spider): + """ + Raises a twisted.internet.error.DNSLookupError: + the default name resolver does not handle IPv6 addresses. + """ name = "ipv6_spider" start_urls = ["http://[::1]"] -process = CrawlerProcess(settings={"RETRY_ENABLED": False}) -process.crawl(IPv6Spider) -process.start() +if __name__ == "__main__": + process = CrawlerProcess(settings={"RETRY_ENABLED": False}) + process.crawl(IPv6Spider) + process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 85035a220..246e54860 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -22,6 +22,8 @@ from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv +from tests.mockserver import MockServer + class BaseCrawlerTest(unittest.TestCase): @@ -280,9 +282,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: - def run_script(self, script_name): + def run_script(self, script_name, *script_args): script_path = os.path.join(self.script_dir, script_name) - args = (sys.executable, script_path) + args = [sys.executable, script_path] + list(script_args) p = subprocess.Popen(args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() @@ -321,11 +323,19 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log) - def test_ipv6_alternative_name_resolver(self): - log = self.run_script('alternative_name_resolver.py') - self.assertIn('Spider closed (finished)', log) + def test_caching_hostname_resolver_ipv6(self): + log = self.run_script("caching_hostname_resolver_ipv6.py") + self.assertIn("Spider closed (finished)", log) self.assertNotIn("twisted.internet.error.DNSLookupError", log) + def test_caching_hostname_resolver_finite_execution(self): + with MockServer() as mock_server: + log = self.run_script("caching_hostname_resolver.py", mock_server.http_address) + self.assertIn("Spider closed (finished)", log) + self.assertNotIn("ERROR: Error downloading", log) + self.assertNotIn("TimeoutError", log) + self.assertNotIn("twisted.internet.error.DNSLookupError", log) + def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") self.assertIn("Spider closed (finished)", log) From 015c82b974841897a637dcd3ddbd40ab48a70ecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 11 Oct 2020 22:09:45 +0200 Subject: [PATCH 43/80] Scrapy 2.4 release notes (#4808) --- docs/news.rst | 306 ++++++++++++++++++++++++++++++++- docs/topics/asyncio.rst | 2 + docs/topics/feed-exports.rst | 18 +- docs/topics/media-pipeline.rst | 31 ++-- docs/topics/settings.rst | 2 +- 5 files changed, 344 insertions(+), 15 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 850b323ef..d5c342c86 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,310 @@ Release notes ============= +.. _release-2.4.0: + +Scrapy 2.4.0 (2020-10-??) +------------------------- + +Highlights: + +* Python 3.5 support has been dropped. + +* The ``file_path`` method of :ref:`media pipelines ` + can now access the source :ref:`item `. + + This allows you to set a download file path based on item data. + +* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows + to define keyword parameters to pass to :ref:`item exporter classes + ` + +* You can now choose whether :ref:`feed exports ` + overwrite or append to the output file. + + For example, when using the :command:`crawl` or :command:`runspider` + commands, you can use the ``-O`` option instead of ``-o`` to overwrite the + output file. + +* Zstd-compressed responses are now supported if zstandard_ is installed. + +* In settings, where the import path of a class is required, it is now + possible to pass a class object instead. + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +* Python 3.6 or greater is now required; support for Python 3.5 has been + dropped + + As a result: + + - When using PyPy, PyPy 7.2.0 or greater :ref:`is now required + ` + + - For Amazon S3 storage support in :ref:`feed exports + ` or :ref:`media pipelines + `, botocore_ 1.4.87 or greater is now required + + - To use the :ref:`images pipeline `, Pillow_ 4.0.0 or + greater is now required + + (:issue:`4718`, :issue:`4732`, :issue:`4733`, :issue:`4742`, :issue:`4743`, + :issue:`4764`) + + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again + discards cookies defined in :attr:`Request.headers + `. + + We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it + was reported that the current implementation could break existing code. + + If you need to set cookies for a request, use the :class:`Request.cookies + ` parameter. + + A future version of Scrapy will include a new, better implementation of the + reverted bug fix. + + (:issue:`4717`, :issue:`4823`) + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* :class:`scrapy.extensions.feedexport.S3FeedStorage` no longer reads the + values of ``access_key`` and ``secret_key`` from the running project + settings when they are not passed to its ``__init__`` method; you must + either pass those parameters to its ``__init__`` method or use + :class:`S3FeedStorage.from_crawler + ` + (:issue:`4356`, :issue:`4411`, :issue:`4688`) + +* :attr:`Rule.process_request ` + no longer admits callables which expect a single ``request`` parameter, + rather than both ``request`` and ``response`` (:issue:`4818`) + + +Deprecations +~~~~~~~~~~~~ + +* In custom :ref:`media pipelines `, signatures that + do not accept a keyword-only ``item`` parameter in any of the methods that + :ref:`now support this parameter ` are now + deprecated (:issue:`4628`, :issue:`4686`) + +* In custom :ref:`feed storage backend classes `, + ``__init__`` method signatures that do not accept a keyword-only + ``feed_options`` parameter are now deprecated (:issue:`547`, :issue:`716`, + :issue:`4512`) + +* The :class:`scrapy.utils.python.WeakKeyCache` class is now deprecated + (:issue:`4684`, :issue:`4701`) + +* The :func:`scrapy.utils.boto.is_botocore` function is now deprecated, use + :func:`scrapy.utils.boto.is_botocore_available` instead (:issue:`4734`, + :issue:`4776`) + + +New features +~~~~~~~~~~~~ + +.. _media-pipeline-item-parameter: + +* The following methods of :ref:`media pipelines ` now + accept an ``item`` keyword-only parameter containing the source + :ref:`item `: + + - In :class:`scrapy.pipelines.files.FilesPipeline`: + + - :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.file_path` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_to_download` + + - In :class:`scrapy.pipelines.images.ImagesPipeline`: + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.file_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.file_path` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.get_images` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.media_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.media_to_download` + + (:issue:`4628`, :issue:`4686`) + +* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows + to define keyword parameters to pass to :ref:`item exporter classes + ` (:issue:`4606`, :issue:`4768`) + +* :ref:`Feed exports ` gained overwrite support: + + * When using the :command:`crawl` or :command:`runspider` commands, you + can use the ``-O`` option instead of ``-o`` to overwrite the output + file + + * You can use the ``overwrite`` key in the :setting:`FEEDS` setting to + configure whether to overwrite the output file (``True``) or append to + its content (``False``) + + * The ``__init__`` and ``from_crawler`` methods of :ref:`feed storage + backend classes ` now receive a new keyword-only + parameter, ``feed_options``, which is a dictionary of :ref:`feed + options ` + + (:issue:`547`, :issue:`716`, :issue:`4512`) + +* Zstd-compressed responses are now supported if zstandard_ is installed + (:issue:`4831`) + +* In settings, where the import path of a class is required, it is now + possible to pass a class object instead (:issue:`3870`, :issue:`3873`). + + This includes also settings where only part of its value is made of an + import path, such as :setting:`DOWNLOADER_MIDDLEWARES` or + :setting:`DOWNLOAD_HANDLERS`. + +* :ref:`Downloader middlewares ` can now + override :class:`response.request `. + + If a :ref:`downloader middleware ` returns + a :class:`~scrapy.http.Response` object from + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` + or + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception` + with a custom :class:`~scrapy.http.Request` object assigned to + :class:`response.request `: + + - The response is handled by the callback of that custom + :class:`~scrapy.http.Request` object, instead of being handled by the + callback of the original :class:`~scrapy.http.Request` object + + - That custom :class:`~scrapy.http.Request` object is now sent as the + ``request`` argument to the :signal:`response_received` signal, instead + of the original :class:`~scrapy.http.Request` object + + (:issue:`4529`, :issue:`4632`) + +* When using the :ref:`FTP feed storage backend `: + + - It is now possible to set the new ``overwrite`` :ref:`feed option + ` to ``False`` to append to an existing file instead of + overwriting it + + - The FTP password can now be omitted if it is not necessary + + (:issue:`547`, :issue:`716`, :issue:`4512`) + +* The ``__init__`` method of :class:`~scrapy.exporters.CsvItemExporter` now + supports an ``errors`` parameter to indicate how to handle encoding errors + (:issue:`4755`) + +* When :ref:`using asyncio `, it is now possible to + :ref:`set a custom asyncio loop ` (:issue:`4306`, + :issue:`4414`) + +* Serialized requests (see :ref:`topics-jobs`) now support callbacks that are + spider methods that delegate on other callable (:issue:`4756`) + +* When a response is larger than :setting:`DOWNLOAD_MAXSIZE`, the logged + message is now a warning, instead of an error (:issue:`3874`, + :issue:`3886`, :issue:`4752`) + + +Bug fixes +~~~~~~~~~ + +* The :command:`genspider` command no longer overwrites existing files + unless the ``--force`` option is used (:issue:`4561`, :issue:`4616`, + :issue:`4623`) + +* Cookies with an empty value are no longer considered invalid cookies + (:issue:`4772`) + +* The :command:`runspider` command now supports files with the ``.pyw`` file + extension (:issue:`4643`, :issue:`4646`) + +* The :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` + middleware now simply ignores unsupported proxy values (:issue:`3331`, + :issue:`4778`) + +* Checks for generator callbacks with a ``return`` statement no longer warn + about ``return`` statements in nested functions (:issue:`4720`, + :issue:`4721`) + +* The system file mode creation mask no longer affects the permissions of + files generated using the :command:`startproject` command (:issue:`4722`) + +* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names + (:issue:`861`, :issue:`4746`) + +* :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can + work when using a headless browser (:issue:`4835`) + + +Documentation +~~~~~~~~~~~~~ + +* The :setting:`FEED_URI_PARAMS` setting is now documented (:issue:`4671`, + :issue:`4724`) + +* Improved the documentation of + :ref:`link extractors ` with an usage example from + a spider callback and reference documentation for the + :class:`~scrapy.link.Link` class (:issue:`4751`, :issue:`4775`) + +* Clarified the impact of :setting:`CONCURRENT_REQUESTS` when using the + :class:`~scrapy.extensions.closespider.CloseSpider` extension + (:issue:`4836`) + +* Removed references to Python 2’s ``unicode`` type (:issue:`4547`, + :issue:`4703`) + +* We now have an :ref:`official deprecation policy ` + (:issue:`4705`) + +* Our :ref:`documentation policies ` now cover usage + of Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged` + directives, and we have removed usages referencing Scrapy 1.4.0 and earlier + versions (:issue:`3971`, :issue:`4310`) + +* Other documentation cleanups (:issue:`4090`, :issue:`4782`, :issue:`4800`, + :issue:`4801`, :issue:`4809`, :issue:`4816`, :issue:`4825`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Extended typing hints (:issue:`4243`, :issue:`4691`) + +* Added tests for the :command:`check` command (:issue:`4663`) + +* Fixed test failures on Debian (:issue:`4726`, :issue:`4727`, :issue:`4735`) + +* Improved Windows test coverage (:issue:`4723`) + +* Switched to :ref:`formatted string literals ` where possible + (:issue:`4307`, :issue:`4324`, :issue:`4672`) + +* Modernized :func:`super` usage (:issue:`4707`) + +* Other code and test cleanups (:issue:`1790`, :issue:`3288`, :issue:`4165`, + :issue:`4564`, :issue:`4651`, :issue:`4714`, :issue:`4738`, :issue:`4745`, + :issue:`4747`, :issue:`4761`, :issue:`4765`, :issue:`4804`, :issue:`4817`, + :issue:`4820`, :issue:`4822`, :issue:`4839`) + + .. _release-2.3.0: Scrapy 2.3.0 (2020-08-04) @@ -4008,9 +4312,9 @@ First release of Scrapy. .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ .. _Twisted: https://twistedmatrix.com/trac/ -.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _w3lib: https://github.com/scrapy/w3lib .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py .. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 .. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/ .. _Zsh: https://www.zsh.org/ +.. _zstandard: https://pypi.org/project/zstandard/ diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index bfb430d52..91e1cca0d 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -1,3 +1,5 @@ +.. _using-asyncio: + ======= asyncio ======= diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 1744cfd74..843ed25f9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -184,7 +184,7 @@ The feeds are stored on `Amazon S3`_. * ``s3://mybucket/path/to/export.csv`` * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` - * Required external libraries: `botocore`_ + * Required external libraries: `botocore`_ >= 1.4.87 The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: @@ -319,6 +319,8 @@ For instance:: }, } +.. _feed-options: + The following is a list of the accepted keys and the setting that is used as a fallback value if that key is not provided for a specific feed definition: @@ -329,6 +331,8 @@ as a fallback value if that key is not provided for a specific feed definition: - ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + .. versionadded:: 2.3.0 + - ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`. - ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. @@ -337,6 +341,8 @@ as a fallback value if that key is not provided for a specific feed definition: - ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. + .. versionadded:: 2.4.0 + - ``overwrite``: whether to overwrite the file if it already exists (``True``) or append to its content (``False``). @@ -355,6 +361,8 @@ as a fallback value if that key is not provided for a specific feed definition: - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) + .. versionadded:: 2.4.0 + - ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`. - ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. @@ -517,7 +525,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. setting:: FEED_EXPORT_BATCH_ITEM_COUNT FEED_EXPORT_BATCH_ITEM_COUNT ------------------------------ +---------------------------- + +.. versionadded:: 2.3.0 Default: ``0`` @@ -586,11 +596,15 @@ The function signature should be as follows: If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id`` is always ``1``. + .. versionadded:: 2.3.0 + - ``batch_time``: UTC date and time, in ISO format with ``:`` replaced with ``-``. See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + .. versionadded:: 2.3.0 + - ``time``: ``batch_time``, with microseconds set to ``0``. :type params: dict diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 06809c24b..156897274 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -56,6 +56,8 @@ this: error will be logged and the file won't be present in the ``files`` field. +.. _images-pipeline: + Using the Images Pipeline ========================= @@ -68,14 +70,10 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to -JPEG/RGB format, so you need to install this library in order to use it. -`Python Imaging Library`_ (PIL) should also work in most cases, but it is known -to cause troubles in some setups, so we recommend to use `Pillow`_ instead of -PIL. +The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for +thumbnailing and normalizing images to JPEG/RGB format. .. _Pillow: https://github.com/python-pillow/Pillow -.. _Python Imaging Library: http://www.pythonware.com/products/pil/ .. _topics-media-pipeline-enabling: @@ -164,14 +162,17 @@ FTP supports two different connection modes: active or passive. Scrapy uses the passive connection mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. +.. _media-pipelines-s3: + Amazon S3 storage ----------------- .. setting:: FILES_STORE_S3_ACL .. setting:: IMAGES_STORE_S3_ACL -:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3 -bucket. Scrapy will automatically upload the files to the bucket. +If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and +:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will +automatically upload the files to the bucket. For example, this is a valid :setting:`IMAGES_STORE` value:: @@ -187,8 +188,9 @@ policy:: For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. -Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like -self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings:: +You can also use other S3-like storages. Storages like self-hosted `Minio`_ or +`s3.scality`_. All you need to do is set endpoint option in you Scrapy +settings:: AWS_ENDPOINT_URL = 'http://minio.example.com:9000' @@ -197,9 +199,10 @@ For self-hosting you also might feel the need not to use SSL and not to verify S AWS_USE_SSL = False # or True (None by default) AWS_VERIFY = False # or True (None by default) +.. _botocore: https://github.com/boto/botocore +.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _Minio: https://github.com/minio/minio .. _s3.scality: https://s3.scality.com/ -.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _media-pipeline-gcs: @@ -446,6 +449,9 @@ See here the methods that you can override in your custom Files Pipeline: By default the :meth:`file_path` method returns ``full/.``. + .. versionadded:: 2.4 + The *item* parameter. + .. method:: FilesPipeline.get_media_requests(item, info) As seen on the workflow, the pipeline will get the URLs of the images to @@ -582,6 +588,9 @@ See here the methods that you can override in your custom Images Pipeline: By default the :meth:`file_path` method returns ``full/.``. + .. versionadded:: 2.4 + The *item* parameter. + .. method:: ImagesPipeline.get_media_requests(item, info) Works the same way as :meth:`FilesPipeline.get_media_requests` method, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 71331c841..912757850 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -102,7 +102,7 @@ module and documented in the :ref:`topics-settings-ref` section. Import paths and classes ======================== -.. versionadded:: VERSION +.. versionadded:: 2.4.0 When a setting references a callable object to be imported by Scrapy, such as a class or a function, there are two different ways you can specify that object: From 47eac8374040c0dc389eb8152a60938dab358bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 11 Oct 2020 22:11:14 +0200 Subject: [PATCH 44/80] Set a release date for Scrapy 2.4.0 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index d5c342c86..a3889705d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.4.0: -Scrapy 2.4.0 (2020-10-??) +Scrapy 2.4.0 (2020-10-11) ------------------------- Highlights: From c340e72988fc6ec615b7b9851c3d28c16c26a839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 11 Oct 2020 22:12:45 +0200 Subject: [PATCH 45/80] =?UTF-8?q?Bump=20version:=202.3.0=20=E2=86=92=202.4?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3c1c8f891..0f142472e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.3.0 +current_version = 2.4.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 276cbf9e2..197c4d5c2 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.3.0 +2.4.0 From 585e4a8aee649f2b439c42d91d857ed1fbdf4fa5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 9 Oct 2020 10:41:19 -0300 Subject: [PATCH 46/80] Replace local server address --- tests/test_crawler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 246e54860..b6de33189 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -330,7 +330,8 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_caching_hostname_resolver_finite_execution(self): with MockServer() as mock_server: - log = self.run_script("caching_hostname_resolver.py", mock_server.http_address) + http_address = mock_server.http_address.replace("0.0.0.0", "127.0.0.1") + log = self.run_script("caching_hostname_resolver.py", http_address) self.assertIn("Spider closed (finished)", log) self.assertNotIn("ERROR: Error downloading", log) self.assertNotIn("TimeoutError", log) From a5872a0fad090c3d3f91f6e03a772265aa50f901 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 30 Oct 2020 20:36:39 +0200 Subject: [PATCH 47/80] Fix output file overwrite with -O (FeedExporter updated) (#4859) --- scrapy/extensions/feedexport.py | 2 +- tests/test_commands.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 7dcb2f52e..3fb4d0e2c 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -452,7 +452,7 @@ class FeedExporter: crawler = getattr(self, 'crawler', None) def build_instance(builder, *preargs): - return build_storage(builder, uri, preargs=preargs) + return build_storage(builder, uri, feed_options=feed_options, preargs=preargs) if crawler and hasattr(feedcls, 'from_crawler'): instance = build_instance(feedcls.from_crawler, crawler) diff --git a/tests/test_commands.py b/tests/test_commands.py index 3e54a0948..85aee55a5 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -680,9 +680,14 @@ class MySpider(scrapy.Spider): ) return [] """ + with open(os.path.join(self.cwd, "example.json"), "w") as f1: + f1.write("not empty") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + with open(os.path.join(self.cwd, "example.json")) as f2: + first_line = f2.readline() + self.assertNotEqual(first_line, "not empty") def test_output_and_overwrite_output(self): spider_code = """ @@ -813,9 +818,14 @@ class MySpider(scrapy.Spider): ) return [] """ + with open(os.path.join(self.cwd, "example.json"), "w") as f1: + f1.write("not empty") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + with open(os.path.join(self.cwd, "example.json")) as f2: + first_line = f2.readline() + self.assertNotEqual(first_line, "not empty") def test_output_and_overwrite_output(self): spider_code = """ From c292957cb19085137146bde72fe82639591c5e1c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 5 Nov 2020 11:15:58 -0300 Subject: [PATCH 48/80] Run Windows tests on GitHub actions (#4869) --- .github/workflows/main.yml | 31 +++++++++++++++++++++++++++++++ azure-pipelines.yml | 22 ---------------------- 2 files changed, 31 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 azure-pipelines.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..28771216c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,31 @@ +name: Run test suite +on: [push, pull_request] + +jobs: + test-windows: + name: "Windows Tests" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest] + python-version: [3.7, 3.8] + env: [TOXENV: py] + include: + - os: windows-latest + python-version: 3.6 + env: + TOXENV: windows-pinned + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Run test suite + env: ${{ matrix.env }} + run: | + pip install -U tox twine wheel codecov + tox diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index c03e258c7..000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,22 +0,0 @@ -variables: - TOXENV: py -pool: - vmImage: 'windows-latest' -strategy: - matrix: - Python36: - python.version: '3.6' - TOXENV: windows-pinned - Python37: - python.version: '3.7' - Python38: - python.version: '3.8' -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - displayName: 'Use Python $(python.version)' -- script: | - pip install -U tox twine wheel codecov - tox - displayName: 'Run test suite' From 5b5478ae9d6f8d5e0028fe47a63252679c457364 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 5 Nov 2020 14:01:34 -0300 Subject: [PATCH 49/80] Call asyncio.get_event_loop when installing the asyncio reactor --- scrapy/utils/reactor.py | 3 +- .../CrawlerProcess/asyncio_deferred_signal.py | 45 +++++++++++++++++++ tests/test_crawler.py | 16 +++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 tests/CrawlerProcess/asyncio_deferred_signal.py diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 831d29462..6723d9b37 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -60,8 +60,9 @@ def install_reactor(reactor_path, event_loop_path=None): if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) else: - event_loop = asyncio.new_event_loop() + event_loop = asyncio.get_event_loop() asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py new file mode 100644 index 000000000..bce300afe --- /dev/null +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -0,0 +1,45 @@ +import asyncio +import sys + +import scrapy + +from scrapy.crawler import CrawlerProcess +from twisted.internet.defer import Deferred + + +class UppercasePipeline: + async def _open_spider(self, spider): + spider.logger.info("async pipeline opened!") + await asyncio.sleep(0.1) + + def open_spider(self, spider): + loop = asyncio.get_event_loop() + return Deferred.fromFuture(loop.create_task(self._open_spider(spider))) + + def process_item(self, item, spider): + return {"url": item["url"].upper()} + + +class UrlSpider(scrapy.Spider): + name = "url_spider" + start_urls = ["data:,"] + custom_settings = { + "ITEM_PIPELINES": {UppercasePipeline: 100}, + } + + def parse(self, response): + yield {"url": response.url} + + +if __name__ == "__main__": + try: + ASYNCIO_EVENT_LOOP = sys.argv[1] + except IndexError: + ASYNCIO_EVENT_LOOP = None + + process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP, + }) + process.crawl(UrlSpider) + process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b6de33189..0faaa79a3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -364,6 +364,22 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) self.assertIn("Using asyncio event loop: uvloop.Loop", log) + @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly") + @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows") + def test_custom_loop_asyncio_deferred_signal(self): + log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + self.assertIn("async pipeline opened!", log) + + def test_default_loop_asyncio_deferred_signal(self): + log = self.run_script("asyncio_deferred_signal.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertNotIn("Using asyncio event loop: uvloop.Loop", log) + self.assertIn("async pipeline opened!", log) + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') From 3095d39740c4818d2c1c98d392255981ebed2a10 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 6 Nov 2020 12:16:10 -0300 Subject: [PATCH 50/80] Test: disable asyncio reactor on Windows for Py>=3.8 --- tests/test_crawler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0faaa79a3..ab113710d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -373,6 +373,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) From 114229eb4a5e3e0289000500cf063518be908d40 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 6 Nov 2020 13:29:14 -0300 Subject: [PATCH 51/80] Docs: add a note about asyncio.set_event_loop --- docs/topics/settings.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 912757850..0086a6c74 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -249,19 +249,25 @@ ASYNCIO_EVENT_LOOP Default: ``None`` -Import path of a given asyncio event loop class. +Import path of a given ``asyncio`` event loop class. -If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the -asyncio event loop to be used with it. Set the setting to the import path of the +If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the +asyncio event loop to be used with it. Set the setting to the import path of the desired asyncio event loop class. If the setting is set to ``None`` the default asyncio event loop will be used. If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor` -function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop -class to be used. +function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop +class to be used. Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`. +.. caution:: Please be aware that, when using a non-default event loop + (either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with + :func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call + :func:`asyncio.set_event_loop`, which will set the specified event loop + as the current loop for the current OS thread. + .. setting:: BOT_NAME BOT_NAME From a2c4a7f9200a06b227a297b9dd2919fe3ec37dbe Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Sun, 8 Nov 2020 19:12:18 -0300 Subject: [PATCH 52/80] Add missing f-string prefix to genspider output --- scrapy/commands/genspider.py | 2 +- tests/test_commands.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 72248bded..5f44daa70 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -98,7 +98,7 @@ class Command(ScrapyCommand): print(f"Created spider {name!r} using template {template_name!r} ", end=('' if spiders_module else '\n')) if spiders_module: - print("in module:\n {spiders_module.__name__}.{module}") + print(f"in module:\n {spiders_module.__name__}.{module}") def _find_template(self, template): template_file = join(self.templates_dir, f'{template}.tmpl') diff --git a/tests/test_commands.py b/tests/test_commands.py index 85aee55a5..d3ac05eac 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -389,8 +389,9 @@ class GenspiderCommandTest(CommandTest): def test_template(self, tplname='crawl'): args = [f'--template={tplname}'] if tplname else [] spname = 'test_spider' + spmodule = f"{self.project_name}.spiders.{spname}" p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module", out) + self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) p, out, err = self.proc('genspider', spname, 'test.com', *args) From 7e98a76ac455a8c69950104766719cde313bbb74 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 9 Nov 2020 12:17:15 -0300 Subject: [PATCH 53/80] Use deferred_from_coro in asyncio test --- tests/CrawlerProcess/asyncio_deferred_signal.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index bce300afe..dd82aa2ff 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -1,9 +1,9 @@ import asyncio import sys -import scrapy - +from scrapy import Spider from scrapy.crawler import CrawlerProcess +from scrapy.utils.defer import deferred_from_coro from twisted.internet.defer import Deferred @@ -14,13 +14,13 @@ class UppercasePipeline: def open_spider(self, spider): loop = asyncio.get_event_loop() - return Deferred.fromFuture(loop.create_task(self._open_spider(spider))) + return deferred_from_coro(self._open_spider(spider)) def process_item(self, item, spider): return {"url": item["url"].upper()} -class UrlSpider(scrapy.Spider): +class UrlSpider(Spider): name = "url_spider" start_urls = ["data:,"] custom_settings = { From b20cfef1e54d9f22769f6d0ec6ae06031bf86ec3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 9 Nov 2020 13:58:52 -0300 Subject: [PATCH 54/80] Remove unnecessary line from test --- tests/CrawlerProcess/asyncio_deferred_signal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index dd82aa2ff..46c2a12a4 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -13,7 +13,6 @@ class UppercasePipeline: await asyncio.sleep(0.1) def open_spider(self, spider): - loop = asyncio.get_event_loop() return deferred_from_coro(self._open_spider(spider)) def process_item(self, item, spider): From c20b34269f488dae4de9433d9c7c783bc481bc6f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 10 Nov 2020 15:35:09 -0300 Subject: [PATCH 55/80] Remove unnecessary pytest-azurepipelines package (#4876) --- tests/requirements-py3.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2eed2f5da..7f8a5c52e 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -6,7 +6,6 @@ mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 -pytest-azurepipelines pytest-cov pytest-twisted >= 1.11 pytest-xdist From 99cc853d6953d336ca65e0eecc0cb3286306bacf Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 29 Sep 2020 23:53:37 -0300 Subject: [PATCH 56/80] Response.protocol attribute --- scrapy/core/downloader/handlers/http11.py | 1 + scrapy/http/response/__init__.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1b041c8a8..0f30b01f9 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -442,6 +442,7 @@ class ScrapyAgent: flags=result["flags"], certificate=result["certificate"], ip_address=result["ip_address"], + protocol=getattr(result["txresponse"], "version", None), ) if result.get("failure"): result["failure"].value.response = response diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c635fde69..f4ef79c72 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -17,8 +17,18 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): - def __init__(self, url, status=200, headers=None, body=b'', flags=None, - request=None, certificate=None, ip_address=None): + def __init__( + self, + url, + status=200, + headers=None, + body=b"", + flags=None, + request=None, + certificate=None, + ip_address=None, + protocol=None, + ): self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) @@ -27,6 +37,7 @@ class Response(object_ref): self.flags = [] if flags is None else list(flags) self.certificate = certificate self.ip_address = ip_address + self.protocol = protocol @property def cb_kwargs(self): @@ -90,7 +101,7 @@ class Response(object_ref): given new values. """ for x in ['url', 'status', 'headers', 'body', - 'request', 'flags', 'certificate', 'ip_address']: + 'request', 'flags', 'certificate', 'ip_address', 'protocol']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) From 5b6b56240c24d02ef69e6cc591ffb2529bc3f36a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:08:37 -0300 Subject: [PATCH 57/80] Test Response.protocol attribute --- tests/test_downloader_handlers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 3e8d7e6b9..eb6d40df7 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -489,6 +489,13 @@ class Http11TestCase(HttpTestCase): def test_download_broken_chunked_content_allow_data_loss_via_setting(self): return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked') + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, (b"HTTP", 1, 1)) + return d + class Https11TestCase(Http11TestCase): scheme = 'https' From 587b4dd71fca12fa5fcc766b891540af77cb27c8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:20:50 -0300 Subject: [PATCH 58/80] Docs for the Response.protocol attribute --- docs/topics/request-response.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f3aaa2c8f..d7d5cd44e 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -693,9 +693,22 @@ Response objects :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` + :param protocol: A tuple containing information about the protocol that was used + to download the response. Taken from the ``version`` attribute of the + corresponding :class:`twisted.web.client.Response` object, it will tipically + consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` + to represent "HTTP/1.1". + :type protocol: :class:`tuple` + + .. versionadded:: 2.X.X + The ``protocol`` parameter. + .. versionadded:: 2.1.0 The ``ip_address`` parameter. + .. versionadded:: 2.0.0 + The ``certificate`` parameter. + .. attribute:: Response.url A string containing the URL of the response. @@ -780,6 +793,8 @@ Response objects .. attribute:: Response.certificate + .. versionadded:: 2.0.0 + A :class:`twisted.internet.ssl.Certificate` object representing the server's SSL certificate. @@ -795,6 +810,20 @@ Response objects handler, i.e. for ``http(s)`` responses. For other handlers, :attr:`ip_address` is always ``None``. + .. attribute:: Response.protocol + + .. versionadded:: 2.X.X + + A tuple containing information about the protocol that was used + to download the response. Taken from the ``version`` attribute of the + corresponding :class:`twisted.web.client.Response` object, it will tipically + consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` + to represent "HTTP/1.1". + + This attribute is currently only populated by the HTTP 1.1 download + handler, i.e. for ``http(s)`` responses. For other handlers, + :attr:`protocol` is always ``None``. + .. method:: Response.copy() Returns a new Response which is a copy of this Response. From 0fb7bcb2cf1606f63f1863bea254a34386c6a0f1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:26:03 -0300 Subject: [PATCH 59/80] Style adjustment --- scrapy/http/response/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f4ef79c72..185a9bb67 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -100,8 +100,9 @@ class Response(object_ref): """Create a new Response with the same attributes except for those given new values. """ - for x in ['url', 'status', 'headers', 'body', - 'request', 'flags', 'certificate', 'ip_address', 'protocol']: + for x in [ + "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol", + ]: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) From 61d089485c7ba66649b936e34833b2013fa12458 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 01:31:15 -0300 Subject: [PATCH 60/80] Docs: sort versionadded directives --- docs/topics/request-response.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d7d5cd44e..f1be41dde 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -700,14 +700,14 @@ Response objects to represent "HTTP/1.1". :type protocol: :class:`tuple` - .. versionadded:: 2.X.X - The ``protocol`` parameter. + .. versionadded:: 2.0.0 + The ``certificate`` parameter. .. versionadded:: 2.1.0 The ``ip_address`` parameter. - .. versionadded:: 2.0.0 - The ``certificate`` parameter. + .. versionadded:: 2.X.X + The ``protocol`` parameter. .. attribute:: Response.url From 22424125560496c9d131c9b7226aaf0f794e5ad8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 10:50:54 -0300 Subject: [PATCH 61/80] Docs: placeholder for versionadded directive --- docs/topics/request-response.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f1be41dde..1cb724227 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -706,7 +706,7 @@ Response objects .. versionadded:: 2.1.0 The ``ip_address`` parameter. - .. versionadded:: 2.X.X + .. versionadded:: VERSION The ``protocol`` parameter. .. attribute:: Response.url @@ -812,7 +812,7 @@ Response objects .. attribute:: Response.protocol - .. versionadded:: 2.X.X + .. versionadded:: VERSION A tuple containing information about the protocol that was used to download the response. Taken from the ``version`` attribute of the From 5e9a99e6a1e940864d9157252592f04658eaf851 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 11:15:29 -0300 Subject: [PATCH 62/80] Reponse.protocol as string --- docs/topics/request-response.rst | 20 +++++++------------- scrapy/core/downloader/handlers/http11.py | 7 ++++++- scrapy/core/downloader/webclient.py | 4 ++-- tests/test_downloader_handlers.py | 9 ++++++++- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1cb724227..98906992d 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -693,12 +693,9 @@ Response objects :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` - :param protocol: A tuple containing information about the protocol that was used - to download the response. Taken from the ``version`` attribute of the - corresponding :class:`twisted.web.client.Response` object, it will tipically - consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` - to represent "HTTP/1.1". - :type protocol: :class:`tuple` + :param protocol: The protocol that was used to download the response. + For instance: "HTTP/1.0", "HTTP/1.1" + :type protocol: :class:`str` .. versionadded:: 2.0.0 The ``certificate`` parameter. @@ -814,14 +811,11 @@ Response objects .. versionadded:: VERSION - A tuple containing information about the protocol that was used - to download the response. Taken from the ``version`` attribute of the - corresponding :class:`twisted.web.client.Response` object, it will tipically - consist of the protocol and version numbers, e.g. ``(b"HTTP", 1, 1)`` - to represent "HTTP/1.1". + The protocol that was used to download the response. + For instance: "HTTP/1.0", "HTTP/1.1" - This attribute is currently only populated by the HTTP 1.1 download - handler, i.e. for ``http(s)`` responses. For other handlers, + This attribute is currently only populated by the HTTP download + handlers, i.e. for ``http(s)`` responses. For other handlers, :attr:`protocol` is always ``None``. .. method:: Response.copy() diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 0f30b01f9..c7553eb87 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -434,6 +434,11 @@ class ScrapyAgent: def _cb_bodydone(self, result, request, url): headers = Headers(result["txresponse"].headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) + try: + version = result["txresponse"].version + protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}" + except (AttributeError, TypeError): + protocol = None response = respcls( url=url, status=int(result["txresponse"].code), @@ -442,7 +447,7 @@ class ScrapyAgent: flags=result["flags"], certificate=result["certificate"], ip_address=result["ip_address"], - protocol=getattr(result["txresponse"], "version", None), + protocol=protocol, ) if result.get("failure"): result["failure"].value.response = response diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index c13683393..9524cce2b 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -7,7 +7,7 @@ from twisted.internet.protocol import ClientFactory from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, to_unicode from scrapy.responsetypes import responsetypes @@ -110,7 +110,7 @@ class ScrapyHTTPClientFactory(ClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) - return respcls(url=self._url, status=status, headers=headers, body=body) + return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version)) def _set_connection_attributes(self, request): parsed = urlparse_cached(request) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index eb6d40df7..a8763a7a5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -360,6 +360,13 @@ class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" download_handler_cls = HTTP10DownloadHandler + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.0") + return d + class Https10TestCase(Http10TestCase): scheme = 'https' @@ -493,7 +500,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL("host"), method="GET") d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, (b"HTTP", 1, 1)) + d.addCallback(self.assertEqual, "HTTP/1.1") return d From b0368228d7f6391c0df41fca1609c6548613ad6b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Nov 2020 11:18:03 -0300 Subject: [PATCH 63/80] Add exception to catch --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c7553eb87..a0fd837b1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -437,7 +437,7 @@ class ScrapyAgent: try: version = result["txresponse"].version protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}" - except (AttributeError, TypeError): + except (AttributeError, TypeError, IndexError): protocol = None response = respcls( url=url, From 85604e1078b927c2a875040e29c58b4594c8d386 Mon Sep 17 00:00:00 2001 From: joaquin garmendia Date: Wed, 11 Nov 2020 15:16:01 -0500 Subject: [PATCH 64/80] Add failed and success count stats to feedstorage backends (#4850) --- scrapy/extensions/feedexport.py | 22 ++++--- tests/test_feedexport.py | 106 ++++++++++++++++++++++++++++---- 2 files changed, 110 insertions(+), 18 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3fb4d0e2c..bec114707 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -319,18 +319,26 @@ class FeedExporter: # Use `largs=log_args` to copy log_args into function's scope # instead of using `log_args` from the outer scope d.addCallback( - lambda _, largs=log_args: logger.info( - logfmt % "Stored", largs, extra={'spider': spider} - ) + self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__ ) d.addErrback( - lambda f, largs=log_args: logger.error( - logfmt % "Error storing", largs, - exc_info=failure_to_exc_info(f), extra={'spider': spider} - ) + self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__ ) return d + def _handle_store_error(self, f, largs, logfmt, spider, slot_type): + logger.error( + logfmt % "Error storing", largs, + exc_info=failure_to_exc_info(f), extra={'spider': spider} + ) + self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") + + def _handle_store_success(self, f, largs, logfmt, spider, slot_type): + logger.info( + logfmt % "Stored", largs, extra={'spider': spider} + ) + self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") + def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): """ Redirect the output data stream to a new file. diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1ea4e8e12..d248824fc 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -8,6 +8,7 @@ import tempfile import warnings from abc import ABC, abstractmethod from collections import defaultdict +from contextlib import ExitStack from io import BytesIO from logging import getLogger from pathlib import Path @@ -47,6 +48,21 @@ from scrapy.utils.test import ( ) from tests.mockserver import MockFTPServer, MockServer +from tests.spiders import ItemSpider + + +def path_to_url(path): + return urljoin('file:', pathname2url(str(path))) + + +def printf_escape(string): + return string.replace('%', '%%') + + +def build_url(path): + if path[0] != '/': + path = '/' + path + return urljoin('file:', path) class FileFeedStorageTest(unittest.TestCase): @@ -620,12 +636,6 @@ class FeedExportTest(FeedExportTestBase): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ - def path_to_url(path): - return urljoin('file:', pathname2url(str(path))) - - def printf_escape(string): - return string.replace('%', '%%') - FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { printf_escape(path_to_url(file_path)): feed_options @@ -748,6 +758,69 @@ class FeedExportTest(FeedExportTestBase): result = self._load_until_eof(data['marshal'], load_func=marshal.load) self.assertEqual(expected, result) + @defer.inlineCallbacks + def test_stats_file_success(self): + settings = { + "FEEDS": { + printf_escape(path_to_url(self._random_temp_filename())): { + "format": "json", + } + }, + } + crawler = get_crawler(ItemSpider, settings) + with MockServer() as mockserver: + yield crawler.crawl(mockserver=mockserver) + self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1) + + @defer.inlineCallbacks + def test_stats_file_failed(self): + settings = { + "FEEDS": { + printf_escape(path_to_url(self._random_temp_filename())): { + "format": "json", + } + }, + } + crawler = get_crawler(ItemSpider, settings) + with ExitStack() as stack: + mockserver = stack.enter_context(MockServer()) + stack.enter_context( + mock.patch( + "scrapy.extensions.feedexport.FileFeedStorage.store", + side_effect=KeyError("foo")) + ) + yield crawler.crawl(mockserver=mockserver) + self.assertIn("feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats()) + self.assertEqual(crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1) + + @defer.inlineCallbacks + def test_stats_multiple_file(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + "FEEDS": { + printf_escape(path_to_url(self._random_temp_filename())): { + "format": "json", + }, + "s3://bucket/key/foo.csv": { + "format": "csv", + }, + "stdout:": { + "format": "xml", + } + }, + } + crawler = get_crawler(ItemSpider, settings) + with MockServer() as mockserver, mock.patch.object(S3FeedStorage, "store"): + yield crawler.crawl(mockserver=mockserver) + self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()) + self.assertIn("feedexport/success_count/S3FeedStorage", crawler.stats.get_stats()) + self.assertIn("feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats()) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1) + @defer.inlineCallbacks def test_export_items(self): # feed exporters use field names from Item @@ -1256,11 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ - def build_url(path): - if path[0] != '/': - path = '/' + path - return urljoin('file:', path) - FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { build_url(file_path): feed @@ -1550,6 +1618,22 @@ class BatchDeliveriesTest(FeedExportTestBase): data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) + @defer.inlineCallbacks + def test_stats_batch_file_success(self): + settings = { + "FEEDS": { + build_url(os.path.join(self._random_temp_filename(), "json", self._file_mark)): { + "format": "json", + } + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + crawler = get_crawler(ItemSpider, settings) + with MockServer() as mockserver: + yield crawler.crawl(total=2, mockserver=mockserver) + self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12) + @defer.inlineCallbacks def test_s3_export(self): skip_if_no_boto() From 2405df49f14cbc052d73e58a819a87417b2502e8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 16 Nov 2020 12:50:33 -0300 Subject: [PATCH 65/80] Add tests for Response.protocol=None --- tests/test_downloader_handlers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a8763a7a5..f51a6cd3c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -115,6 +115,7 @@ class FileTestCase(unittest.TestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.status, 200) self.assertEqual(response.body, b'0123456789') + self.assertEqual(response.protocol, None) request = Request(path_to_file_uri(self.tmpname + '^')) assert request.url.upper().endswith('%5E') @@ -976,6 +977,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'I have the power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) + self.assertIsNone(r.protocol) return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): @@ -1134,3 +1136,10 @@ class DataURITestCase(unittest.TestCase): request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') return self.download_request(request, self.spider).addCallback(_test) + + def test_protocol(self): + def _test(response): + self.assertIsNone(response.protocol) + + request = Request("data:,") + return self.download_request(request, self.spider).addCallback(_test) From 15d301e968aa3e26a28f771cf1b45635e84ef094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Nov 2020 09:16:08 +0100 Subject: [PATCH 66/80] Cover Scrapy 2.4.1 in the release notes (#4884) Co-authored-by: Mikhail Korobov --- docs/news.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index a3889705d..e92493252 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,29 @@ Release notes ============= +.. _release-2.4.1: + +Scrapy 2.4.1 (2020-11-17) +------------------------- + +- Fixed :ref:`feed exports ` overwrite support (:issue:`4845`, :issue:`4857`, :issue:`4859`) + +- Fixed the AsyncIO event loop handling, which could make code hang + (:issue:`4855`, :issue:`4872`) + +- Fixed the IPv6-capable DNS resolver + :class:`~scrapy.resolver.CachingHostnameResolver` for download handlers + that call + :meth:`reactor.resolve ` + (:issue:`4802`, :issue:`4803`) + +- Fixed the output of the :command:`genspider` command showing placeholders + instead of the import part of the generated spider module (:issue:`4874`) + +- Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`, + :issue:`4876`) + + .. _release-2.4.0: Scrapy 2.4.0 (2020-10-11) From 26836c4e1ae9588ee173c5977fc6611364ca7cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Nov 2020 09:17:39 +0100 Subject: [PATCH 67/80] =?UTF-8?q?Bump=20version:=202.4.0=20=E2=86=92=202.4?= =?UTF-8?q?.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0f142472e..956c512cb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.0 +current_version = 2.4.1 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 197c4d5c2..005119baa 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.4.0 +2.4.1 From 63becd1bc89395750b39139e2114193607f3ca61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Nov 2020 21:58:08 +0100 Subject: [PATCH 68/80] Update news.rst --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index e92493252..0391506c4 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -20,7 +20,7 @@ Scrapy 2.4.1 (2020-11-17) (:issue:`4802`, :issue:`4803`) - Fixed the output of the :command:`genspider` command showing placeholders - instead of the import part of the generated spider module (:issue:`4874`) + instead of the import path of the generated spider module (:issue:`4874`) - Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`, :issue:`4876`) From 075ab156b0ac7bf56828fd40a843f3e9b63a3de3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 11:59:59 -0300 Subject: [PATCH 69/80] Deprecate scrapy.utils.py36 module --- scrapy/utils/py36.py | 17 +++++++++-------- scrapy/utils/python.py | 7 +++++++ scrapy/utils/spider.py | 7 ++----- setup.cfg | 3 --- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py index c8c24076e..070b145a4 100644 --- a/scrapy/utils/py36.py +++ b/scrapy/utils/py36.py @@ -1,10 +1,11 @@ -""" -Helpers using Python 3.6+ syntax (ignore SyntaxError on import). -""" +import warnings + +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.python import collect_asyncgen # noqa: F401 -async def collect_asyncgen(result): - results = [] - async for x in result: - results.append(x) - return results +warnings.warn( + "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.python` instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, +) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5703fd4c3..7bd286523 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -355,3 +355,10 @@ class MutableChain: @deprecated("scrapy.utils.python.MutableChain.__next__") def next(self): return self.__next__() + + +async def collect_asyncgen(result): + results = [] + async for x in result: + results.append(x) + return results diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index f3a9a67a3..5319604ba 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,17 +4,14 @@ import logging from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter -try: - from scrapy.utils.py36 import collect_asyncgen -except SyntaxError: - collect_asyncgen = None +from scrapy.utils.python import collect_asyncgen logger = logging.getLogger(__name__) def iterate_spider_output(result): - if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result): + if inspect.isasyncgen(result): d = deferred_from_coro(collect_asyncgen(result)) d.addCallback(iterate_spider_output) return d diff --git a/setup.cfg b/setup.cfg index 8101443e3..0c9f6b963 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,9 +55,6 @@ ignore_errors = True [mypy-scrapy.utils.response] ignore_errors = True -[mypy-scrapy.utils.spider] -ignore_errors = True - [mypy-scrapy.utils.trackref] ignore_errors = True From 4075e1eadd0a9e83356d18b9ca73090e4a0bc770 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 15:07:56 -0300 Subject: [PATCH 70/80] Remove deprecated modules (utils.http/markup/multipart) --- pytest.ini | 3 --- scrapy/utils/http.py | 36 ------------------------------------ scrapy/utils/markup.py | 14 -------------- scrapy/utils/multipart.py | 15 --------------- 4 files changed, 68 deletions(-) delete mode 100644 scrapy/utils/http.py delete mode 100644 scrapy/utils/markup.py delete mode 100644 scrapy/utils/multipart.py diff --git a/pytest.ini b/pytest.ini index 1c95f715a..d4deeb57c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -36,8 +36,5 @@ flake8-ignore = scrapy/spiders/__init__.py E402 F401 # Issues pending a review: - scrapy/utils/http.py F403 - scrapy/utils/markup.py F403 - scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 tests/test_loader.py E741 diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py deleted file mode 100644 index ceb3f0509..000000000 --- a/scrapy/utils/http.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.http instead of this module -""" - -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.decorators import deprecated -from w3lib.http import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.http` is deprecated, " - "Please import from `w3lib.http` instead.", - ScrapyDeprecationWarning, stacklevel=2) - - -@deprecated -def decode_chunked_transfer(chunked_body): - """Parsed body received with chunked transfer encoding, and return the - decoded body. - - For more info see: - https://en.wikipedia.org/wiki/Chunked_transfer_encoding - - """ - body, h, t = '', '', chunked_body - while t: - h, t = t.split('\r\n', 1) - if h == '0': - break - size = int(h, 16) - body += t[:size] - t = t[size + 2:] - return body diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py deleted file mode 100644 index 9728c542a..000000000 --- a/scrapy/utils/markup.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.html instead of this module -""" -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.html import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.markup` is deprecated. " - "Please import from `w3lib.html` instead.", - ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py deleted file mode 100644 index 5dcf791b8..000000000 --- a/scrapy/utils/multipart.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.form instead of this module -""" -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.form import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.multipart` is deprecated. " - "If you're using `encode_multipart` function, please use " - "`urllib3.filepost.encode_multipart_formdata` instead", - ScrapyDeprecationWarning, stacklevel=2) From 51ca4d0138e5c9cf637074f59c839ff9b5839db6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 15:47:08 -0300 Subject: [PATCH 71/80] Remove deprecated scrapy.utils.gz.is_gzipped function --- scrapy/utils/gz.py | 15 +----------- tests/test_utils_gz.py | 55 ++++++++++++------------------------------ 2 files changed, 16 insertions(+), 54 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 11d433cf5..76156a4b8 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,7 +1,6 @@ +import struct from gzip import GzipFile from io import BytesIO -import re -import struct from scrapy.utils.decorators import deprecated @@ -42,17 +41,5 @@ def gunzip(data): return b''.join(output_list) -_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search -_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search - - -@deprecated -def is_gzipped(response): - """Return True if the response is gzipped, or False otherwise""" - ctype = response.headers.get('Content-Type', b'') - cenc = response.headers.get('Content-Encoding', b'').lower() - return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip') - - def gzip_magic_number(response): return response.body[:3] == b'\x1f\x8b\x08' diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 7148185f4..4943731cb 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -3,10 +3,11 @@ from os.path import join from w3lib.encoding import html_to_unicode -from scrapy.utils.gz import gunzip, is_gzipped -from scrapy.http import Response, Headers +from scrapy.utils.gz import gunzip, gzip_magic_number +from scrapy.http import Response from tests import tests_datadir + SAMPLEDIR = join(tests_datadir, 'compressed') @@ -14,8 +15,12 @@ class GunzipTest(unittest.TestCase): def test_gunzip_basic(self): with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f: - text = gunzip(f.read()) - self.assertEqual(len(text), 9950) + r1 = Response("http://www.example.com", body=f.read()) + self.assertTrue(gzip_magic_number(r1)) + + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + self.assertFalse(gzip_magic_number(r2)) + self.assertEqual(len(r2.body), 9950) def test_gunzip_truncated(self): with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f: @@ -28,46 +33,16 @@ class GunzipTest(unittest.TestCase): def test_gunzip_truncated_short(self): with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f: - text = gunzip(f.read()) - assert text.endswith(b'') + r1 = Response("http://www.example.com", body=f.read()) + self.assertTrue(gzip_magic_number(r1)) - def test_is_x_gzipped_right(self): - hdrs = Headers({"Content-Type": "application/x-gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - def test_is_gzipped_right(self): - hdrs = Headers({"Content-Type": "application/gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - def test_is_gzipped_not_quite(self): - hdrs = Headers({"Content-Type": "application/gzippppp"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_case_insensitive(self): - hdrs = Headers({"Content-Type": "Application/X-Gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - hdrs = Headers({"Content-Type": "application/X-GZIP ; charset=utf-8"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert r2.body.endswith(b'') + self.assertFalse(gzip_magic_number(r2)) def test_is_gzipped_empty(self): r1 = Response("http://www.example.com") - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_wrong(self): - hdrs = Headers({"Content-Type": "application/javascript"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_with_charset(self): - hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) + self.assertFalse(gzip_magic_number(r1)) def test_gunzip_illegal_eof(self): with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f: From 462014bc5738a5ed18ec4c15a91384eb17e57096 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 15:51:59 -0300 Subject: [PATCH 72/80] Scheduler: remove support for deprecated queuelib.PriorityQueue --- scrapy/core/scheduler.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index a18c26b17..9ce823dbc 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,14 +1,10 @@ import os import json import logging -import warnings from os.path import join, exists -from queuelib import PriorityQueue - from scrapy.utils.misc import load_object, create_instance from scrapy.utils.job import job_dir -from scrapy.utils.deprecate import ScrapyDeprecationWarning logger = logging.getLogger(__name__) @@ -56,14 +52,6 @@ class Scheduler: dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) dupefilter = create_instance(dupefilter_cls, settings, crawler) pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) - if pqclass is PriorityQueue: - warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'" - " is no longer supported because of API changes; " - "please use 'scrapy.pqueues.ScrapyPriorityQueue'", - ScrapyDeprecationWarning) - from scrapy.pqueues import ScrapyPriorityQueue - pqclass = ScrapyPriorityQueue - dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) logunser = settings.getbool('SCHEDULER_DEBUG') From 0a93df9efd20a4bcd34d2ee5e6bdf6d365d70409 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 16:16:18 -0300 Subject: [PATCH 73/80] Move collect_asyncgen to utils.asyncgen --- scrapy/utils/asyncgen.py | 5 +++++ scrapy/utils/py36.py | 4 ++-- scrapy/utils/python.py | 7 ------- 3 files changed, 7 insertions(+), 9 deletions(-) create mode 100644 scrapy/utils/asyncgen.py diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py new file mode 100644 index 000000000..7f697af5f --- /dev/null +++ b/scrapy/utils/asyncgen.py @@ -0,0 +1,5 @@ +async def collect_asyncgen(result): + results = [] + async for x in result: + results.append(x) + return results diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py index 070b145a4..653e2bbbb 100644 --- a/scrapy/utils/py36.py +++ b/scrapy/utils/py36.py @@ -1,11 +1,11 @@ import warnings from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.python import collect_asyncgen # noqa: F401 +from scrapy.utils.asyncgen import collect_asyncgen # noqa: F401 warnings.warn( - "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.python` instead.", + "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.", category=ScrapyDeprecationWarning, stacklevel=2, ) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7bd286523..5703fd4c3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -355,10 +355,3 @@ class MutableChain: @deprecated("scrapy.utils.python.MutableChain.__next__") def next(self): return self.__next__() - - -async def collect_asyncgen(result): - results = [] - async for x in result: - results.append(x) - return results From 18b05af87783d71f9c8f9f1ebe027083037e6f86 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 16:18:58 -0300 Subject: [PATCH 74/80] Remove tests/test_utils_http.py --- tests/test_utils_http.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 tests/test_utils_http.py diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py deleted file mode 100644 index 363b015a8..000000000 --- a/tests/test_utils_http.py +++ /dev/null @@ -1,19 +0,0 @@ -import unittest - -from scrapy.utils.http import decode_chunked_transfer - - -class ChunkedTest(unittest.TestCase): - - def test_decode_chunked_transfer(self): - """Example taken from: http://en.wikipedia.org/wiki/Chunked_transfer_encoding""" - chunked_body = "25\r\n" + "This is the data in the first chunk\r\n\r\n" - chunked_body += "1C\r\n" + "and this is the second one\r\n\r\n" - chunked_body += "3\r\n" + "con\r\n" - chunked_body += "8\r\n" + "sequence\r\n" - chunked_body += "0\r\n\r\n" - body = decode_chunked_transfer(chunked_body) - self.assertEqual( - body, - "This is the data in the first chunk\r\nand this is the second one\r\nconsequence" - ) From fe8bb6bd905ad2a733ba5e876f4197888605902e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Nov 2020 16:51:04 -0300 Subject: [PATCH 75/80] Fix import --- scrapy/utils/spider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 5319604ba..59fc9202f 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,7 +4,7 @@ import logging from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import collect_asyncgen +from scrapy.utils.asyncgen import collect_asyncgen logger = logging.getLogger(__name__) From 95d39d5cb464ca22516a30f96c0a323613421090 Mon Sep 17 00:00:00 2001 From: etimoz Date: Sun, 29 Nov 2020 13:24:04 +0100 Subject: [PATCH 76/80] removed wrong super argument in overriding serialize_fields code example --- docs/topics/exporters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index ef50c9f5c..0a0a1765a 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -123,7 +123,7 @@ Example:: def serialize_field(self, field, name, value): if field == 'price': return f'$ {str(value)}' - return super(Product, self).serialize_field(field, name, value) + return super().serialize_field(field, name, value) .. _topics-exporters-reference: From 7fec9f991f0bd415900df2bf288c6cda909ecd77 Mon Sep 17 00:00:00 2001 From: Kader DJEHAF Date: Mon, 30 Nov 2020 21:47:28 +0100 Subject: [PATCH 77/80] [Cleaned] PEP 8: E251 unexpected spaces around keyword / parameter equals (#4911) [Cleaned] PEP 8: E251 unexpected spaces around keyword / parameter equals --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0c2281400..35736b75f 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( name='Scrapy', version=version, url='https://scrapy.org', - project_urls = { + project_urls={ 'Documentation': 'https://docs.scrapy.org/', 'Source': 'https://github.com/scrapy/scrapy', 'Tracker': 'https://github.com/scrapy/scrapy/issues', From a80bafe5cdcb4b5e2542524fe641570f9107a121 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 30 Nov 2020 19:03:13 -0300 Subject: [PATCH 78/80] Remove deprecated SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE --- scrapy/utils/project.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index fd13d85e3..35e59a258 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -68,18 +68,10 @@ def get_project_settings(): if settings_module_path: settings.setmodule(settings_module_path, priority='project') - pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") - if pickled_settings: - warnings.warn("Use of environment variable " - "'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE' " - "is deprecated.", ScrapyDeprecationWarning) - settings.setdict(pickle.loads(pickled_settings), priority='project') - scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if k.startswith('SCRAPY_')} valid_envvars = { 'CHECK', - 'PICKLED_SETTINGS_TO_OVERRIDE', 'PROJECT', 'PYTHON_SHELL', 'SETTINGS_MODULE', From 6091f3cc03835b24def08bf358d7b19207be685d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Dec 2020 10:26:21 -0300 Subject: [PATCH 79/80] Remove unused pickle import --- scrapy/utils/project.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 35e59a258..c66af497e 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,5 +1,4 @@ import os -import pickle import warnings from importlib import import_module From db10aaf9eb858c8deb35e7ef96538549a8e36be0 Mon Sep 17 00:00:00 2001 From: gunadhya <6939749+gunadhya@users.noreply.github.com> Date: Thu, 3 Dec 2020 15:26:36 +0530 Subject: [PATCH 80/80] Update links in installation guide (#4899) --- docs/conf.py | 1 + docs/intro/install.rst | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 27d2b5dff..543507a46 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -283,6 +283,7 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { 'attrs': ('https://www.attrs.org/en/stable/', None), 'coverage': ('https://coverage.readthedocs.io/en/stable', None), + 'cryptography' : ('https://cryptography.io/en/latest/', None), 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), 'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None), 'pytest': ('https://docs.pytest.org/en/latest', None), diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 3bfd3bc3b..73d7ede42 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -69,10 +69,9 @@ In case of any trouble related to these dependencies, please refer to their respective installation instructions: * `lxml installation`_ -* `cryptography installation`_ +* :doc:`cryptography installation ` .. _lxml installation: https://lxml.de/installation.html -.. _cryptography installation: https://cryptography.io/en/latest/installation/ .. _intro-using-virtualenv: @@ -265,7 +264,6 @@ For details, see `Issue #2473 `_. .. _cryptography: https://cryptography.io/en/latest/ .. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/ .. _setuptools: https://pypi.python.org/pypi/setuptools -.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ .. _Scrapinghub: https://scrapinghub.com