From c49b5aaf77a668e44bffb54c52310084e4068162 Mon Sep 17 00:00:00 2001 From: zaid-ismail031 Date: Thu, 25 Aug 2022 02:17:03 +0200 Subject: [PATCH 01/51] Changed incorrect information regarding the return type of parse/request callback method. --- docs/topics/spiders.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index ece02ae47..ffe41cf3e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -181,9 +181,10 @@ scrapy.Spider scraped data and/or more URLs to follow. Other Requests callbacks have the same requirements as the :class:`Spider` class. - This method, as well as any other Request callback, must return an - iterable of :class:`~scrapy.Request` and/or :ref:`item objects - `. + This method, as well as any other Request callback, must return a + :class:`~scrapy.Request` object, an :ref:`item object `, an + iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects + `, or ``None``. :param response: the response to parse :type response: :class:`~scrapy.http.Response` From ce0ca51485545aed8eb33a285bc3d2fbf8a8e407 Mon Sep 17 00:00:00 2001 From: "Magsen (CD)" Date: Tue, 13 Sep 2022 12:07:58 +0200 Subject: [PATCH 02/51] fix: typo in tutorial fix: typo in tutorial --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 75928077e..092123d1d 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -379,7 +379,7 @@ like this: Let's open up scrapy shell and play a bit to find out how to extract the data we want:: - $ scrapy shell 'https://quotes.toscrape.com' + scrapy shell 'https://quotes.toscrape.com' We get a list of selectors for the quote HTML elements with: From 1429aa011ce2a7a43691fc359c88bf40e512176c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Tue, 20 Sep 2022 12:47:20 -0400 Subject: [PATCH 03/51] Update test-standard link in contributing docs (#5631) --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 946bdc23e..9cfe10012 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -214,7 +214,7 @@ Tests ===== Tests are implemented using the :doc:`Twisted unit-testing framework -`. Running tests requires +`. Running tests requires :doc:`tox `. .. _running-tests: From 5f194202114fd38530c78299d51b6966b4802f59 Mon Sep 17 00:00:00 2001 From: Tim B Date: Wed, 21 Sep 2022 07:27:27 +0100 Subject: [PATCH 04/51] Documented how settings must be picklable (#5629) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves Co-authored-by: Laerte Pereira <5853172+Laerte@users.noreply.github.com> --- docs/topics/settings.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 6722ce9ed..90f13f3ef 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -98,6 +98,10 @@ class. The global defaults are located in the ``scrapy.settings.default_settings`` module and documented in the :ref:`topics-settings-ref` section. +Compatibility with pickle +========================= + +Setting values must be :ref:`picklable `. Import paths and classes ======================== From 385acd5598fb06a126ac485087b58a4581096876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sat, 24 Sep 2022 14:58:14 -0400 Subject: [PATCH 05/51] Match pyOpenSSL and service_identity to Twisted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- setup.py | 6 +++--- tests/test_crawler.py | 2 +- tox.ini | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index ed197273f..fcc902c62 100644 --- a/setup.py +++ b/setup.py @@ -20,13 +20,13 @@ def has_environment_marker_platform_impl_support(): install_requires = [ 'Twisted>=18.9.0', - 'cryptography>=2.8', + 'cryptography>=3.3', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'pyOpenSSL>=19.1.0', + 'pyOpenSSL>=21.0.0', 'queuelib>=1.4.2', - 'service_identity>=16.0.0', + 'service_identity>=18.1.0', 'w3lib>=1.17.0', 'zope.interface>=5.1.0', 'protego>=0.1.15', diff --git a/tests/test_crawler.py b/tests/test_crawler.py index e2a14be55..cf15ba9b9 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -327,7 +327,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_reactor_default_twisted_reactor_select(self): log = self.run_script('reactor_default_twisted_reactor_select.py') - if platform.system() == 'Windows': + if platform.system() in ['Windows', 'Darwin']: # The goal of this test function is to test that, when a reactor is # installed (the default one here) and a different reactor is # configured (select here), an error raises. diff --git a/tox.ini b/tox.ini index 2110e1020..2bf9454d0 100644 --- a/tox.ini +++ b/tox.ini @@ -73,15 +73,15 @@ commands = [pinned] deps = - cryptography==2.8 + cryptography==3.3 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - pyOpenSSL==19.1.0 + pyOpenSSL==21.0.0 queuelib==1.4.2 - service_identity==16.0.0 + service_identity==18.1.0 Twisted[http2]==18.9.0 w3lib==1.17.0 zope.interface==5.1.0 From 79a4bc3da02bfe2b07c596f6b677760b2c04f96f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sun, 25 Sep 2022 14:17:57 -0400 Subject: [PATCH 06/51] Cleanup METHOD_SSLv3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- docs/topics/settings.rst | 1 - scrapy/core/downloader/contextfactory.py | 2 +- scrapy/core/downloader/tls.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 90f13f3ef..a711fd197 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -564,7 +564,6 @@ This setting must be one of these string values: set this if you want the behavior of Scrapy<1.1 - ``'TLSv1.1'``: forces TLS version 1.1 - ``'TLSv1.2'``: forces TLS version 1.2 -- ``'SSLv3'``: forces SSL version 3 (**not recommended**) .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index b5318c7bb..4abde2238 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,7 +21,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): which allows TLS protocol negotiation 'A TLS/SSL connection established with [this method] may - understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.' + understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.' """ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs): diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 19a56d9b6..698a1c85c 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -11,7 +11,6 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info logger = logging.getLogger(__name__) -METHOD_SSLv3 = 'SSLv3' METHOD_TLS = 'TLS' METHOD_TLSv10 = 'TLSv1.0' METHOD_TLSv11 = 'TLSv1.1' @@ -20,7 +19,6 @@ METHOD_TLSv12 = 'TLSv1.2' openssl_methods = { METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended) METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only From 1d79994dccf396caca93f53cd50646281fee1def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 27 Sep 2022 17:01:33 +0200 Subject: [PATCH 07/51] Copy 2.6.3 release notes from the 2.6 branch --- docs/news.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index d27c105a5..9469d0fe5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,32 @@ Release notes ============= +.. _release-2.6.3: + +Scrapy 2.6.3 (2022-09-27) +------------------------- + +- Added support for pyOpenSSL_ 22.1.0, removing support for SSLv3 + (:issue:`5634`, :issue:`5635`, :issue:`5636`). + +- Upgraded the minimum versions of the following dependencies: + + - cryptography_: 2.0 → 3.3 + + - pyOpenSSL_: 16.2.0 → 21.0.0 + + - service_identity_: 16.0.0 → 18.1.0 + + - Twisted_: 17.9.0 → 18.9.0 + + - zope.interface_: 4.1.3 → 5.0.0 + + (:issue:`5621`, :issue:`5632`) + +- Fixes test and documentation issues (:issue:`5612`, :issue:`5617`, + :issue:`5631`). + + .. _release-2.6.2: Scrapy 2.6.2 (2022-07-25) From 3ca7877781fdae5415c57a9a0f348b1797795209 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 29 Sep 2022 11:51:11 -0300 Subject: [PATCH 08/51] chore: Skip `batch_path_differ` test on Windows --- tests/test_feedexport.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ecd1b59d3..ad2383018 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2285,6 +2285,7 @@ class BatchDeliveriesTest(FeedExportTestBase): for expected_batch, got_batch in zip(expected, data[fmt]): self.assertEqual(expected_batch, got_batch) + @pytest.mark.skipif(sys.platform == 'win32', reason='Odd behaviour on file creation/output') @defer.inlineCallbacks def test_batch_path_differ(self): """ @@ -2305,7 +2306,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) - self.assertEqual(len(items) + 1, len(data['json'])) + self.assertEqual(len(items), len([_ for _ in data['json'] if _])) @defer.inlineCallbacks def test_stats_batch_file_success(self): From 9f006e3aa595acc6499f08a90696d7cd0dc0bca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sun, 25 Sep 2022 13:48:51 -0400 Subject: [PATCH 09/51] Match pyOpenSSL and types-pyOpenSSL versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2bf9454d0..0a47a1c0b 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ basepython = python3 deps = lxml-stubs==0.2.0 mypy==0.971 - types-pyOpenSSL==20.0.3 + types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 commands = mypy --show-error-codes {posargs: scrapy tests} From 116d9a97481f5ee4028a1f3f8e72ca34b35a0be9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Fri, 30 Sep 2022 22:58:14 -0400 Subject: [PATCH 10/51] Correct distutils deprecation warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- scrapy/utils/display.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index f4d17224b..d28df40c7 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -5,7 +5,7 @@ pprint and pformat wrappers with colorization support import ctypes import platform import sys -from distutils.version import LooseVersion as parse_version +from packaging.version import Version as parse_version from pprint import pformat as pformat_ From c3f35d2ad79d19e7e000b8ba0df7d1d9f10658b2 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sat, 1 Oct 2022 19:37:51 +0200 Subject: [PATCH 11/51] ci(tests-windows): upgrade actions/setup-python to v4 --- .github/workflows/tests-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 955b9b449..0d85a8e09 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 9fcbf3bcbc8b8d09bb7d8b246dae9156d6848a5b Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sat, 1 Oct 2022 19:38:16 +0200 Subject: [PATCH 12/51] ci(tests-windows): upgrade actions/checkout to v3 --- .github/workflows/tests-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 955b9b449..2d8c140af 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -25,7 +25,7 @@ jobs: TOXENV: asyncio steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 From 1a2cb61e22506c47e78bc800739d9cb2a8bf0144 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sat, 1 Oct 2022 20:27:10 +0200 Subject: [PATCH 13/51] ci(test-macos): upgrade actions/checkout to v3 --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 7819a4e12..8c9ef5c2f 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -10,7 +10,7 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 From af95331296a91444947095af19f8244d8bc915e1 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sat, 1 Oct 2022 20:27:23 +0200 Subject: [PATCH 14/51] ci(test-macos): upgrade actions/setup-python to v4 --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 7819a4e12..d49272fbf 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 759ad5dee4120dc2197ea6fa1559c92f77472abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sun, 2 Oct 2022 09:09:04 -0400 Subject: [PATCH 15/51] Require packaging --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index fcc902c62..8b9c43738 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'setuptools', + 'packaging', 'tldextract', 'lxml>=4.3.0', ] From c7d800ab229df50c2767ad67460e90f6213fc293 Mon Sep 17 00:00:00 2001 From: Abdul Rauf Date: Sun, 2 Oct 2022 19:12:48 +0500 Subject: [PATCH 16/51] CI: add Twine check in check workflow --- .github/workflows/checks.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b26f344ff..a708474ef 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -39,3 +39,8 @@ jobs: run: | pip install -U tox tox + - name: Twine check + run: | + pip install twine + python setup.py sdist + twine check dist/* From 69bf5c662555db64979a90c5c284b1faa4f24992 Mon Sep 17 00:00:00 2001 From: Abdul Rauf Date: Sun, 2 Oct 2022 20:27:24 +0500 Subject: [PATCH 17/51] CI: move twinecheck to tox env --- .github/workflows/checks.yml | 8 +++----- tox.ini | 8 ++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a708474ef..e515959ad 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -25,6 +25,9 @@ jobs: - python-version: "3.10" # Keep in sync with .readthedocs.yml env: TOXENV: docs + - python-version: "3.10" + env: + TOXENV: twinecheck steps: - uses: actions/checkout@v2 @@ -39,8 +42,3 @@ jobs: run: | pip install -U tox tox - - name: Twine check - run: | - pip install twine - python setup.py sdist - twine check dist/* diff --git a/tox.ini b/tox.ini index 2bf9454d0..2d94ba78f 100644 --- a/tox.ini +++ b/tox.ini @@ -71,6 +71,14 @@ deps = commands = pylint conftest.py docs extras scrapy setup.py tests +[testenv:twinecheck] +basepython = python3 +deps = + twine==4.0.1 +commands = + python setup.py sdist + twine check dist/* + [pinned] deps = cryptography==3.3 From 80194f1c0374b2aa429de4c7ea70a683c946db95 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 2 Oct 2022 15:22:06 -0300 Subject: [PATCH 18/51] CrawlSpider: add support for async def callbacks --- scrapy/spiders/crawl.py | 6 ++++-- tests/spiders.py | 12 ++++++++++++ tests/test_crawl.py | 11 +++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 1dcf2e6ab..d860ae0b4 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,7 +6,7 @@ See documentation in docs/topics/spiders.rst """ import copy -from typing import Sequence +from typing import Awaitable, Sequence from scrapy.http import Request, HtmlResponse from scrapy.linkextractors import LinkExtractor @@ -109,9 +109,11 @@ class CrawlSpider(Spider): rule = self._rules[failure.request.meta['rule']] return self._handle_failure(failure, rule.errback) - def _parse_response(self, response, callback, cb_kwargs, follow=True): + async def _parse_response(self, response, callback, cb_kwargs, follow=True): if callback: cb_res = callback(response, **cb_kwargs) or () + if isinstance(cb_res, Awaitable): + cb_res = await cb_res cb_res = self.process_results(response, cb_res) for request_or_item in iterate_spider_output(cb_res): yield request_or_item diff --git a/tests/spiders.py b/tests/spiders.py index 3b69aa7ae..2b78e1f7c 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -369,6 +369,18 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"}) +class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod): + """A CrawlSpider with an async def callback""" + name = 'crawl_spider_with_async_callback' + rules = ( + Rule(LinkExtractor(), callback='parse_async', follow=True), + ) + + async def parse_async(self, response, foo=None): + self.logger.info('[parse_async] status %i (foo: %s)', response.status, foo) + return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"}) + + class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = 'crawl_spider_with_errback' rules = ( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index c11871745..d14021319 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -37,6 +37,7 @@ from tests.spiders import ( BrokenStartRequestsSpider, BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, + CrawlSpiderWithAsyncCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, DelaySpider, @@ -391,6 +392,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[parse] status 201 (foo: None)", str(log)) self.assertIn("[parse] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks + def test_crawlspider_with_async_callback(self): + crawler = get_crawler(CrawlSpiderWithAsyncCallback) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse_async] status 200 (foo: None)", str(log)) + self.assertIn("[parse_async] status 201 (foo: None)", str(log)) + self.assertIn("[parse_async] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback) From da8f915091a769b3c3aca0f60d2370c2240213a1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 2 Oct 2022 17:37:10 -0300 Subject: [PATCH 19/51] Adapt for asyng generator callbacks --- scrapy/spiders/crawl.py | 11 +++++++---- scrapy/utils/asyncgen.py | 2 +- tests/spiders.py | 12 ++++++++++++ tests/test_crawl.py | 11 +++++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d860ae0b4..edac082d0 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,11 +6,12 @@ See documentation in docs/topics/spiders.rst """ import copy -from typing import Awaitable, Sequence +from typing import AsyncIterable, Awaitable, Sequence -from scrapy.http import Request, HtmlResponse +from scrapy.http import Request, Response, HtmlResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider +from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.spider import iterate_spider_output @@ -78,7 +79,7 @@ class CrawlSpider(Spider): def parse_start_url(self, response, **kwargs): return [] - def process_results(self, response, results): + def process_results(self, response: Response, results: list): return results def _build_request(self, rule_index, link): @@ -112,7 +113,9 @@ class CrawlSpider(Spider): async def _parse_response(self, response, callback, cb_kwargs, follow=True): if callback: cb_res = callback(response, **cb_kwargs) or () - if isinstance(cb_res, Awaitable): + if isinstance(cb_res, AsyncIterable): + cb_res = await collect_asyncgen(cb_res) + elif isinstance(cb_res, Awaitable): cb_res = await cb_res cb_res = self.process_results(response, cb_res) for request_or_item in iterate_spider_output(cb_res): diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 9f794de92..c84b51e8c 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,7 +1,7 @@ from typing import AsyncGenerator, AsyncIterable, Iterable, Union -async def collect_asyncgen(result: AsyncIterable): +async def collect_asyncgen(result: AsyncIterable) -> list: results = [] async for x in result: results.append(x) diff --git a/tests/spiders.py b/tests/spiders.py index 2b78e1f7c..5ea8a4a21 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -381,6 +381,18 @@ class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod): return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"}) +class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod): + """A CrawlSpider with an async generator callback""" + name = 'crawl_spider_with_async_generator_callback' + rules = ( + Rule(LinkExtractor(), callback='parse_async_gen', follow=True), + ) + + async def parse_async_gen(self, response, foo=None): + self.logger.info('[parse_async_gen] status %i (foo: %s)', response.status, foo) + yield Request(self.mockserver.url("/status?n=202"), self.parse_async_gen, cb_kwargs={"foo": "bar"}) + + class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = 'crawl_spider_with_errback' rules = ( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d14021319..8be4b6fe1 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -38,6 +38,7 @@ from tests.spiders import ( BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, + CrawlSpiderWithAsyncGeneratorCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, DelaySpider, @@ -402,6 +403,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[parse_async] status 201 (foo: None)", str(log)) self.assertIn("[parse_async] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks + def test_crawlspider_with_async_generator_callback(self): + crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log)) + self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log)) + self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback) From 41041ae740260c4bddda06ab7f4d8e4351e6e0e4 Mon Sep 17 00:00:00 2001 From: Felipe A Date: Mon, 3 Oct 2022 00:48:12 -0300 Subject: [PATCH 20/51] refact: add Osx DS_Store file to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index d77d24624..6c5c50e08 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ test-output.* # Windows Thumbs.db + +# OSX miscellaneous +.DS_Store \ No newline at end of file From 82d10f09145186fbd3a2c445287a7bf3e1e54263 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 6 Oct 2022 20:27:06 +0600 Subject: [PATCH 21/51] Add Ubuntu tests for Python 3.11rc2. --- .github/workflows/tests-ubuntu.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index be40c7c71..9e62b8e23 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -20,6 +20,12 @@ jobs: - python-version: "3.10" env: TOXENV: asyncio + - python-version: "3.11.0-rc.2" + env: + TOXENV: py + - python-version: "3.11.0-rc.2" + env: + TOXENV: asyncio - python-version: pypy3 env: TOXENV: pypy3 @@ -53,7 +59,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4' + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' run: | sudo apt-get update # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version From fa58ab21e4670db1b11b6097d809cd18dd2ca667 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 6 Oct 2022 20:59:06 +0600 Subject: [PATCH 22/51] Replace _getargspec_py23() with inspect.getfullargspec(). --- scrapy/utils/python.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 11c089ac2..8ce030d9d 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -180,23 +180,6 @@ def binary_is_text(data): return all(c not in _BINARYCHARS for c in data) -def _getargspec_py23(func): - """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords, - defaults) - - Was identical to inspect.getargspec() in python2, but uses - inspect.getfullargspec() for python3 behind the scenes to avoid - DeprecationWarning. - - >>> def f(a, b=2, *ar, **kw): - ... pass - - >>> _getargspec_py23(f) - ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,)) - """ - return inspect.ArgSpec(*inspect.getfullargspec(func)[:4]) - - def get_func_args(func, stripself=False): """Return the argument name list of a callable""" if inspect.isfunction(func): @@ -248,9 +231,9 @@ def get_spec(func): """ if inspect.isfunction(func) or inspect.ismethod(func): - spec = _getargspec_py23(func) + spec = inspect.getfullargspec(func) elif hasattr(func, '__call__'): - spec = _getargspec_py23(func.__call__) + spec = inspect.getfullargspec(func.__call__) else: raise TypeError(f'{type(func)} is not callable') From e60e8224a23ddd4fd98c54318d128f6744f40a6a Mon Sep 17 00:00:00 2001 From: Abinash Satapathy Date: Thu, 6 Oct 2022 19:58:48 +0200 Subject: [PATCH 23/51] Update and rename INSTALL to INSTALL.md --- INSTALL | 4 ---- INSTALL.md | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 INSTALL create mode 100644 INSTALL.md diff --git a/INSTALL b/INSTALL deleted file mode 100644 index 06e812936..000000000 --- a/INSTALL +++ /dev/null @@ -1,4 +0,0 @@ -For information about installing Scrapy see: - -* docs/intro/install.rst (local file) -* https://docs.scrapy.org/en/latest/intro/install.html (online version) diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 000000000..495413f97 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,4 @@ +For information about installing Scrapy see: + +* [Local docs](docs/intro/install.rst) +* [Online docs](https://docs.scrapy.org/en/latest/intro/install.html) From 300c42bfdf0afe0809b03cf571ebe5da7352301e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 14:53:19 +0600 Subject: [PATCH 24/51] Install PyPy using actions/setup-python. --- .github/workflows/tests-ubuntu.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 9e62b8e23..633e01c95 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -26,10 +26,9 @@ jobs: - python-version: "3.11.0-rc.2" env: TOXENV: asyncio - - python-version: pypy3 + - python-version: pypy3.9 env: TOXENV: pypy3 - PYPY_VERSION: 3.9-v7.3.9 # pinned deps - python-version: 3.7.13 @@ -38,10 +37,9 @@ jobs: - python-version: 3.7.13 env: TOXENV: asyncio-pinned - - python-version: pypy3 + - python-version: pypy3.7 env: TOXENV: pypy3-pinned - PYPY_VERSION: 3.7-v7.3.5 # extras # extra-deps includes reppy, which does not support Python 3.9 @@ -59,7 +57,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' run: | sudo apt-get update # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version @@ -68,13 +66,6 @@ jobs: - name: Run tests env: ${{ matrix.env }} run: | - if [[ ! -z "$PYPY_VERSION" ]]; then - export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi pip install -U tox tox From 4aea925714f7c9beb68d3d478ad528f00d488942 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 15:01:50 +0600 Subject: [PATCH 25/51] Update action versions. --- .github/workflows/tests-ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 633e01c95..9cd0df468 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -49,10 +49,10 @@ jobs: TOXENV: extra-deps steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 424849b27582d3a74fab0c43d842576722b48e25 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 15:17:55 +0600 Subject: [PATCH 26/51] Update mypy. --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2804ebe95..12a4516ce 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,8 @@ install_command = basepython = python3 deps = lxml-stubs==0.2.0 - mypy==0.971 + mypy==0.982 + types-attrs==19.1.0 types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 commands = From ccb6a8c098501c0f4322faeac17f9b97dad0c5e3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 15:27:14 +0600 Subject: [PATCH 27/51] Add a note about flake8. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 12a4516ce..7002c2f18 100644 --- a/tox.ini +++ b/tox.ini @@ -59,6 +59,7 @@ deps = # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 pytest-flake8==1.1.1 + # newer ones don't work: https://github.com/tholo/pytest-flake8/issues/87 flake8==4.0.1 commands = pytest --flake8 {posargs:docs scrapy tests} From d1515cc0755ae1d587ffbe4171544442fa10ef8d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 15:30:44 +0600 Subject: [PATCH 28/51] Update pylint. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 7002c2f18..463828e7c 100644 --- a/tox.ini +++ b/tox.ini @@ -69,7 +69,7 @@ commands = basepython = python3.8 deps = {[testenv:extra-deps]deps} - pylint==2.14.5 + pylint==2.15.3 commands = pylint conftest.py docs extras scrapy setup.py tests From ed9bc84d551c7302cd075aa52f30d38465749f05 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 8 Oct 2022 19:11:16 +0600 Subject: [PATCH 29/51] Remove a pin on libxml2-dev. --- .github/workflows/tests-ubuntu.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 9cd0df468..7915a9aab 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -60,8 +60,7 @@ jobs: if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' run: | sudo apt-get update - # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version - sudo apt-get install libxml2-dev/bionic-updates libxslt-dev + sudo apt-get install libxml2-dev libxslt-dev - name: Run tests env: ${{ matrix.env }} From 92f2d75ed39e4d75f6cca23b8d3aaf934677f4f3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 8 Oct 2022 19:12:32 +0600 Subject: [PATCH 30/51] Add a classifier for Python 3.11. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8b9c43738..a43cf08c8 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ setup( 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', From 5fa613b419f0d94b7dca8ae4ba2782ca268a9d7c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Oct 2022 11:02:57 +0600 Subject: [PATCH 31/51] Run flake8 directly. --- conftest.py | 10 ---------- tox.ini | 3 +-- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/conftest.py b/conftest.py index 117087790..d7fe80321 100644 --- a/conftest.py +++ b/conftest.py @@ -42,16 +42,6 @@ def chdir(tmpdir): tmpdir.chdir() -def pytest_collection_modifyitems(session, config, items): - # Avoid executing tests when executing `--flake8` flag (pytest-flake8) - try: - from pytest_flake8 import Flake8Item - if config.getoption('--flake8'): - items[:] = [item for item in items if isinstance(item, Flake8Item)] - except ImportError: - pass - - def pytest_addoption(parser): parser.addoption( "--reactor", diff --git a/tox.ini b/tox.ini index 463828e7c..822e96fde 100644 --- a/tox.ini +++ b/tox.ini @@ -58,11 +58,10 @@ deps = {[testenv]deps} # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 - pytest-flake8==1.1.1 # newer ones don't work: https://github.com/tholo/pytest-flake8/issues/87 flake8==4.0.1 commands = - pytest --flake8 {posargs:docs scrapy tests} + flake8 {posargs:docs scrapy tests} [testenv:pylint] # reppy does not support Python 3.9+ From eeb199adda2d7b56c78879df79c7294148ca15f1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Oct 2022 11:10:59 +0600 Subject: [PATCH 32/51] Fix flake8 issues in previously ignored files. --- .flake8 | 2 ++ docs/_ext/scrapydocs.py | 24 +++++++++---------- scrapy/utils/testsite.py | 2 +- .../CrawlerProcess/asyncio_deferred_signal.py | 1 - .../CrawlerProcess/asyncio_enabled_reactor.py | 4 ++-- tests/CrawlerProcess/reactor_default.py | 3 +-- .../reactor_default_twisted_reactor_select.py | 4 +--- tests/CrawlerProcess/reactor_select.py | 2 -- ..._select_subclass_twisted_reactor_select.py | 4 ---- .../reactor_select_twisted_reactor_select.py | 3 --- 10 files changed, 19 insertions(+), 30 deletions(-) diff --git a/.flake8 b/.flake8 index 1c503fb0b..d7aebc24b 100644 --- a/.flake8 +++ b/.flake8 @@ -4,6 +4,8 @@ max-line-length = 119 ignore = W503 exclude = + docs/conf.py + # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file # F401: Module imported but unused diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 640660943..d02a2e17b 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -80,24 +80,24 @@ def replace_settingslist_nodes(app, doctree, fromdocname): def setup(app): app.add_crossref_type( - directivename = "setting", - rolename = "setting", - indextemplate = "pair: %s; setting", + directivename="setting", + rolename="setting", + indextemplate="pair: %s; setting", ) app.add_crossref_type( - directivename = "signal", - rolename = "signal", - indextemplate = "pair: %s; signal", + directivename="signal", + rolename="signal", + indextemplate="pair: %s; signal", ) app.add_crossref_type( - directivename = "command", - rolename = "command", - indextemplate = "pair: %s; command", + directivename="command", + rolename="command", + indextemplate="pair: %s; command", ) app.add_crossref_type( - directivename = "reqmeta", - rolename = "reqmeta", - indextemplate = "pair: %s; reqmeta", + directivename="reqmeta", + rolename="reqmeta", + indextemplate="pair: %s; reqmeta", ) app.add_role('source', source_role) app.add_role('commit', commit_role) diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index fce77be32..5d3710391 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -23,7 +23,7 @@ class NoMetaRefreshRedirect(util.Redirect): def render(self, request): content = util.Redirect.render(self, request) return content.replace(b'http-equiv=\"refresh\"', - b'http-no-equiv=\"do-not-refresh-me\"') + b'http-no-equiv=\"do-not-refresh-me\"') def test_site(): diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index bdd3c1fef..b83f6a585 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -5,7 +5,6 @@ from typing import Optional from scrapy import Spider from scrapy.crawler import CrawlerProcess from scrapy.utils.defer import deferred_from_coro -from twisted.internet.defer import Deferred class UppercasePipeline: diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index f2a93074b..e561d63c7 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -6,8 +6,8 @@ if sys.version_info >= (3, 8) and sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) -import scrapy -from scrapy.crawler import CrawlerProcess +import scrapy # noqa: E402 +from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 5a21a3717..2c867df61 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,6 +1,6 @@ import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import reactor +from twisted.internet import reactor # noqa: F401 class NoRequestsSpider(scrapy.Spider): @@ -14,4 +14,3 @@ process = CrawlerProcess(settings={}) process.crawl(NoRequestsSpider) process.start() - diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index c476722ef..c2b30b044 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -1,6 +1,6 @@ import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import reactor +from twisted.internet import reactor # noqa: F401 class NoRequestsSpider(scrapy.Spider): @@ -16,5 +16,3 @@ process = CrawlerProcess(settings={ process.crawl(NoRequestsSpider) process.start() - - diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index eac6e2f89..ca70c06a0 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -15,5 +15,3 @@ process = CrawlerProcess(settings={}) process.crawl(NoRequestsSpider) process.start() - - diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 47f480605..0035daf1e 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -25,7 +25,3 @@ process = CrawlerProcess(settings={ process.crawl(NoRequestsSpider) process.start() - - - - diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index e0d2dab26..4f8394edb 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -17,6 +17,3 @@ process = CrawlerProcess(settings={ process.crawl(NoRequestsSpider) process.start() - - - From 5bf42606792c69268cc34da287293e9406e25883 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Oct 2022 11:14:20 +0600 Subject: [PATCH 33/51] Update flake8. --- tests/test_http_response.py | 6 ++++-- tox.ini | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 5d67a5e74..b42c95045 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -705,7 +705,8 @@ class HtmlResponseTest(TextResponseTest): def test_html_encoding(self): - body = b"""Some page + body = b"""Some page + Price: \xa3100' """ r1 = self.response_class("http://www.example.com", body=body) @@ -719,7 +720,8 @@ class HtmlResponseTest(TextResponseTest): self._assert_response_values(r2, 'iso-8859-1', body) # for conflicting declarations headers must take precedence - body = b"""Some page + body = b"""Some page + Price: \xa3100' """ r3 = self.response_class("http://www.example.com", body=body, diff --git a/tox.ini b/tox.ini index 822e96fde..eee99cb2d 100644 --- a/tox.ini +++ b/tox.ini @@ -58,8 +58,7 @@ deps = {[testenv]deps} # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 - # newer ones don't work: https://github.com/tholo/pytest-flake8/issues/87 - flake8==4.0.1 + flake8==5.0.4 commands = flake8 {posargs:docs scrapy tests} From b792632046093504b1df8bcb0b92d4cb52ebf436 Mon Sep 17 00:00:00 2001 From: Nirjas Jakilim Date: Mon, 10 Oct 2022 11:47:02 +0600 Subject: [PATCH 34/51] updated setup-python and checkout workflow --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e515959ad..cc8f20f44 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -30,10 +30,10 @@ jobs: TOXENV: twinecheck steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 44b682830..8e307189d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,10 +7,10 @@ jobs: if: startsWith(github.event.ref, 'refs/tags/') steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: "3.10" From d12fcc555b57dbd62c90212bd1d918d39e1ba45d Mon Sep 17 00:00:00 2001 From: Derek Date: Tue, 11 Oct 2022 10:32:45 -0700 Subject: [PATCH 35/51] Link to the Code of Conduct (#5659) --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 358302c76..d416ced3c 100644 --- a/README.rst +++ b/README.rst @@ -95,8 +95,7 @@ See https://docs.scrapy.org/en/master/contributing.html for details. Code of Conduct --------------- -Please note that this project is released with a Contributor Code of Conduct -(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). +Please note that this project is released with a Contributor `Code of Conduct `_. By participating in this project you agree to abide by its terms. Please report unacceptable behavior to opensource@zyte.com. From 96fb663ae1a95286b0634bc51bc17b42eb29bd0f Mon Sep 17 00:00:00 2001 From: Abdul Rauf Date: Tue, 11 Oct 2022 10:34:18 -0700 Subject: [PATCH 36/51] README: set Bash highlighting for pip install (#5648) --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d416ced3c..970bf2c35 100644 --- a/README.rst +++ b/README.rst @@ -64,7 +64,9 @@ Requirements Install ======= -The quick way:: +The quick way: + +.. code:: bash pip install scrapy From da9a2f8a946d6341be1cdd8cad0616a18bba6dbe Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 12 Oct 2022 11:10:39 -0300 Subject: [PATCH 37/51] Remove mention of minimum PyPy versions from the documentation (#5678) --- docs/intro/install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 80a9c16d6..f28f5216a 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -10,7 +10,7 @@ Supported Python versions ========================= Scrapy requires Python 3.7+, either the CPython implementation (default) or -the PyPy 7.3.5+ implementation (see :ref:`python:implementations`). +the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: @@ -219,7 +219,7 @@ After any of these workarounds you should be able to install Scrapy:: PyPy ---- -We recommend using the latest PyPy version. The version tested is 5.9.0. +We recommend using the latest PyPy version. For PyPy3, only Linux installation was tested. Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. From 715c05d504d22e87935ae42cee55ee35b12c2ebd Mon Sep 17 00:00:00 2001 From: gabrielztk <38334108+gabrielztk@users.noreply.github.com> Date: Thu, 13 Oct 2022 07:22:10 -0300 Subject: [PATCH 38/51] =?UTF-8?q?transport.producer.loseConnection()=20?= =?UTF-8?q?=E2=86=92=20transport.loseConnection()=20(#4995)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/core/downloader/handlers/http11.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 38935667d..6b8a18f1a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -384,8 +384,7 @@ class ScrapyAgent: logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": request, "handler": handler.__qualname__}) txresponse._transport.stopProducing() - with suppress(AttributeError): - txresponse._transport._producer.loseConnection() + txresponse._transport.loseConnection() return { "txresponse": txresponse, "body": b"", @@ -417,7 +416,7 @@ class ScrapyAgent: logger.warning(warning_msg, warning_args) - txresponse._transport._producer.loseConnection() + txresponse._transport.loseConnection() raise defer.CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: @@ -543,7 +542,7 @@ class _ResponseReader(protocol.Protocol): logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": self._request, "handler": handler.__qualname__}) self.transport.stopProducing() - self.transport._producer.loseConnection() + self.transport.loseConnection() failure = result if result.value.fail else None self._finish_response(flags=["download_stopped"], failure=failure) From 62cc26e209b66ce5941f15736f669c75dcac9a59 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Oct 2022 22:03:54 +0600 Subject: [PATCH 39/51] Change TWISTED_REACTOR in the default template. --- docs/topics/settings.rst | 5 +++++ scrapy/templates/project/module/settings.py.tmpl | 1 + 2 files changed, 6 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a711fd197..0b1ef71cf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1642,6 +1642,11 @@ install the default reactor defined by Twisted for the current platform. This is to maintain backward compatibility and avoid possible problems caused by using a non-default reactor. +.. versionchanged:: VERSION + The :command:`startproject` command now sets this setting to + ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated + ``settings.py`` file. + For additional information, see :doc:`core/howto/choosing-reactor`. diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 5e541e2c0..c0c34e986 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -89,3 +89,4 @@ ROBOTSTXT_OBEY = True # Set settings whose default value is deprecated to a future-proof value REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION' +TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' From 22a59d0005c03d866ce98d352024974a9e48e7d1 Mon Sep 17 00:00:00 2001 From: Nirjas Jakilim Date: Fri, 14 Oct 2022 23:41:50 +0600 Subject: [PATCH 40/51] CI: use the latest version of Ubuntu (#5675) --- .github/workflows/checks.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index cc8f20f44..439dfee51 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: checks: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8e307189d..f6b098b80 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,7 +3,7 @@ on: [push] jobs: publish: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest if: startsWith(github.event.ref, 'refs/tags/') steps: diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 7915a9aab..d2bfe4a5f 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: tests: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: From 043575123c57db9f055c3668a3d80485989df0b2 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sat, 15 Oct 2022 11:41:05 +0330 Subject: [PATCH 41/51] Add async callback support to the parse command (#5577) --- scrapy/commands/parse.py | 55 ++++++++++++++++++++++--------------- tests/test_command_parse.py | 15 ++++++++++ 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 9b4fb0ed6..d93ab2ac5 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -5,6 +5,8 @@ from typing import Dict from itemadapter import is_item, ItemAdapter from w3lib.url import is_url +from twisted.internet.defer import maybeDeferred + from scrapy.commands import BaseRunSpiderCommand from scrapy.http import Request from scrapy.utils import display @@ -110,16 +112,19 @@ class Command(BaseRunSpiderCommand): if not opts.nolinks: self.print_requests(colour=colour) - def run_callback(self, response, callback, cb_kwargs=None): - cb_kwargs = cb_kwargs or {} + def _get_items_and_requests(self, spider_output, opts, depth, spider, callback): items, requests = [], [] - - for x in iterate_spider_output(callback(response, **cb_kwargs)): + for x in spider_output: if is_item(x): items.append(x) elif isinstance(x, Request): requests.append(x) - return items, requests + return items, requests, opts, depth, spider, callback + + def run_callback(self, response, callback, cb_kwargs=None): + cb_kwargs = cb_kwargs or {} + d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs)) + return d def get_callback_from_rules(self, spider, response): if getattr(spider, 'rules', None): @@ -158,6 +163,25 @@ class Command(BaseRunSpiderCommand): logger.error('No response downloaded for: %(url)s', {'url': url}) + def scraped_data(self, args): + items, requests, opts, depth, spider, callback = args + if opts.pipelines: + itemproc = self.pcrawler.engine.scraper.itemproc + for item in items: + itemproc.process_item(item, spider) + self.add_items(depth, items) + self.add_requests(depth, requests) + + scraped_data = items if opts.output else [] + if depth < opts.depth: + for req in requests: + req.meta['_depth'] = depth + 1 + req.meta['_callback'] = req.callback + req.callback = callback + scraped_data += requests + + return scraped_data + def prepare_request(self, spider, request, opts): def callback(response, **cb_kwargs): # memorize first request @@ -191,23 +215,10 @@ class Command(BaseRunSpiderCommand): # parse items and requests depth = response.meta['_depth'] - items, requests = self.run_callback(response, cb, cb_kwargs) - if opts.pipelines: - itemproc = self.pcrawler.engine.scraper.itemproc - for item in items: - itemproc.process_item(item, spider) - self.add_items(depth, items) - self.add_requests(depth, requests) - - scraped_data = items if opts.output else [] - if depth < opts.depth: - for req in requests: - req.meta['_depth'] = depth + 1 - req.meta['_callback'] = req.callback - req.callback = callback - scraped_data += requests - - return scraped_data + d = self.run_callback(response, cb, cb_kwargs) + d.addCallback(self._get_items_and_requests, opts, depth, spider, callback) + d.addCallback(self.scraped_data) + return d # update request meta if any extra meta was passed through the --meta/-m opts. if opts.meta: diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 0d992be56..8368356e2 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -29,7 +29,15 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule +from scrapy.utils.test import get_from_asyncio_queue +class AsyncDefAsyncioSpider(scrapy.Spider): + + name = 'asyncdef{self.spider_name}' + + async def parse(self, response): + status = await get_from_asyncio_queue(response.status) + return [scrapy.Item(), dict(foo='bar')] class MySpider(scrapy.Spider): name = '{self.spider_name}' @@ -160,6 +168,13 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url('/html')]) self.assertIn("INFO: It Works!", _textmode(stderr)) + @defer.inlineCallbacks + def test_asyncio_parse_items(self): + status, out, stderr = yield self.execute( + ['--spider', 'asyncdef' + self.spider_name, '-c', 'parse', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + @defer.inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( From 75bb516edbc39f9a657e71e75f05f0fd6a33d60d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 15 Oct 2022 10:26:38 +0200 Subject: [PATCH 42/51] Adapt tests to the new value of TWISTED_REACTOR for new projects --- tests/test_commands.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 76d5f3935..eaca41102 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -689,8 +689,15 @@ class MySpider(scrapy.Spider): ]) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - def test_asyncio_enabled_false(self): + def test_asyncio_enabled_default(self): log = self.get_log(self.debug_log_spider, args=[]) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + def test_asyncio_enabled_false(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', 'TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor' + ]) + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') From 960a7f68f6939744b39d528e28f8f925a2e12ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 15 Oct 2022 11:27:00 +0200 Subject: [PATCH 43/51] Verify that the installed asyncio event loop matches ASYNCIO_EVENT_LOOP (#5529) Co-authored-by: Laerte Pereira <5853172+Laerte@users.noreply.github.com> --- scrapy/crawler.py | 14 ++++++++-- scrapy/utils/reactor.py | 18 ++++++++++++ .../asyncio_enabled_reactor_different_loop.py | 25 +++++++++++++++++ .../asyncio_enabled_reactor_same_loop.py | 28 +++++++++++++++++++ tests/test_crawler.py | 23 +++++++++++++++ 5 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py create mode 100644 tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e768bca12..65174d846 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -31,7 +31,12 @@ from scrapy.utils.log import ( ) from scrapy.utils.misc import create_instance, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names -from scrapy.utils.reactor import install_reactor, verify_installed_reactor +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + verify_installed_asyncio_event_loop, + verify_installed_reactor, +) logger = logging.getLogger(__name__) @@ -78,17 +83,20 @@ class Crawler: crawler=self, ) - reactor_class = self.settings.get("TWISTED_REACTOR") + reactor_class = self.settings["TWISTED_REACTOR"] + event_loop = self.settings["ASYNCIO_EVENT_LOOP"] if init_reactor: # this needs to be done after the spider settings are merged, # but before something imports twisted.internet.reactor if reactor_class: - install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) + install_reactor(reactor_class, event_loop) else: from twisted.internet import reactor # noqa: F401 log_reactor_info() if reactor_class: verify_installed_reactor(reactor_class) + if is_asyncio_reactor_installed() and event_loop: + verify_installed_asyncio_event_loop(event_loop) self.extensions = ExtensionManager.from_crawler(self) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index bc543b230..652733ce8 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -90,6 +90,24 @@ def verify_installed_reactor(reactor_path): raise Exception(msg) +def verify_installed_asyncio_event_loop(loop_path): + from twisted.internet import reactor + loop_class = load_object(loop_path) + if isinstance(reactor._asyncioEventloop, loop_class): + return + installed = ( + f"{reactor._asyncioEventloop.__class__.__module__}" + f".{reactor._asyncioEventloop.__class__.__qualname__}" + ) + specified = f"{loop_class.__module__}.{loop_class.__qualname__}" + raise Exception( + "Scrapy found an asyncio Twisted reactor already " + f"installed, and its event loop class ({installed}) does " + "not match the one specified in the ASYNCIO_EVENT_LOOP " + f"setting ({specified})" + ) + + def is_asyncio_reactor_installed(): from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py new file mode 100644 index 000000000..ea8242f67 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -0,0 +1,25 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncioreactor.install(asyncio.get_event_loop()) + +import scrapy # noqa: E402 +from scrapy.crawler import CrawlerProcess # noqa: E402 + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py new file mode 100644 index 000000000..d24bf3031 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -0,0 +1,28 @@ +import asyncio +import sys + +from uvloop import Loop + +from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncio.set_event_loop(Loop()) +asyncioreactor.install(asyncio.get_event_loop()) + +import scrapy # noqa: E402 +from scrapy.crawler import CrawlerProcess # noqa: E402 + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index cf15ba9b9..c61d461f7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -454,6 +454,29 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + def test_asyncio_enabled_reactor_same_loop(self): + log = self.run_script("asyncio_enabled_reactor_same_loop.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + def test_asyncio_enabled_reactor_different_loop(self): + log = self.run_script("asyncio_enabled_reactor_different_loop.py") + self.assertNotIn("Spider closed (finished)", log) + self.assertIn( + ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ), + log, + ) + def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) From c49764ffd7111d8a465a0d077d0df38bd40449fa Mon Sep 17 00:00:00 2001 From: mattkohl-flex Date: Mon, 17 Oct 2022 11:15:17 +0100 Subject: [PATCH 44/51] typo fixes --- docs/intro/install.rst | 2 +- docs/topics/contracts.rst | 2 +- docs/topics/extensions.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/request-response.rst | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index f28f5216a..9ab479edd 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -187,7 +187,7 @@ solutions: * Install `homebrew`_ following the instructions in https://brew.sh/ * Update your ``PATH`` variable to state that homebrew packages should be - used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly + used before system packages (Change ``.bashrc`` to ``.zshrc`` accordingly if you're using `zsh`_ as default shell):: echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index ef296dc9e..c29a3a410 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -102,7 +102,7 @@ override three methods: .. method:: Contract.post_process(output) This allows processing the output of the callback. Iterators are - converted listified before being passed to this hook. + converted to lists before being passed to this hook. Raise :class:`~scrapy.exceptions.ContractFail` from :class:`~scrapy.contracts.Contract.pre_process` or diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 297e1fdc5..130657b0b 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -17,7 +17,7 @@ settings, just like any other Scrapy code. It is customary for extensions to prefix their settings with their own name, to avoid collision with existing (and future) extensions. For example, a -hypothetic extension to handle `Google Sitemaps`_ would use settings like +hypothetical extension to handle `Google Sitemaps`_ would use settings like ``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on. .. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 477652704..33441838a 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -154,7 +154,7 @@ Too many spiders? If your project has too many spiders executed in parallel, the output of :func:`prefs()` can be difficult to read. For this reason, that function has a ``ignore`` argument which can be used to -ignore a particular class (and all its subclases). For +ignore a particular class (and all its subclasses). For example, this won't show any live references to spiders: >>> from scrapy.spiders import Spider diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 49cb69f67..7eb6942ac 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -446,7 +446,7 @@ class). Scenarios where changing the request fingerprinting algorithm may cause undesired results include, for example, using the HTTP cache middleware (see :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`). -Changing the request fingerprinting algorithm would invalidade the current +Changing the request fingerprinting algorithm would invalidate the current cache, requiring you to redownload all requests again. Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in From 06c8f673afe9af08784e62d67930a8cbc9887ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 17 Oct 2022 15:04:29 +0200 Subject: [PATCH 45/51] 2.7 release notes (#5680) * Fix the display name of documented fingerprinter class methods * Initial draft for the Scrapy 2.7 release notes * Update VERSION and PREVIOUS_VERSION references * Clarify the restrictions lifted for item field output names * Fix the description of the BOM bug fix * Fix the note about changes in MIME sniffing * Fix typo * Extend highlights * Fyx typo --- docs/_ext/scrapydocs.py | 2 +- docs/news.rst | 193 +++++++++++++++++- docs/topics/components.rst | 4 +- docs/topics/coroutines.rst | 12 +- docs/topics/request-response.rst | 45 ++-- docs/topics/settings.rst | 10 +- docs/topics/spider-middleware.rst | 8 +- scrapy/settings/default_settings.py | 2 +- .../templates/project/module/settings.py.tmpl | 2 +- scrapy/utils/request.py | 12 +- scrapy/utils/test.py | 2 +- tests/test_crawl.py | 2 +- tests/test_crawler.py | 8 +- tests/test_dupefilters.py | 12 +- tests/test_pipeline_crawl.py | 2 +- tests/test_scheduler.py | 2 +- tests/test_spiderloader/__init__.py | 2 +- tests/test_utils_request.py | 4 +- 18 files changed, 259 insertions(+), 65 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index d02a2e17b..f0f382da3 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -15,7 +15,7 @@ class SettingsListDirective(Directive): def is_setting_index(node): - if node.tagname == 'index': + if node.tagname == 'index' and node['entries']: # index entries for setting directives look like: # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] entry_type, info, refid = node['entries'][0][:3] diff --git a/docs/news.rst b/docs/news.rst index 9469d0fe5..d8b9fcd1e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,195 @@ Release notes ============= +.. _release-2.7.0: + +Scrapy 2.7.0 (to be released) +----------------------------- + +Highlights: + +- Added Python 3.11 support, dropped Python 3.6 support +- Improved support for :ref:`asynchronous callbacks ` +- :ref:`Asyncio support ` is enabled by default on new + projects +- Output names of item fields can now be arbitrary strings +- Centralized :ref:`request fingerprinting ` + configuration is now possible + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +Python 3.7 or greater is now required; support for Python 3.6 has been dropped. +Support for the upcoming Python 3.11 has been added. + +The minimum required version of some dependencies has changed as well: + +- lxml_: 3.5.0 → 4.3.0 + +- Pillow_ (:ref:`images pipeline `): 4.0.0 → 7.1.0 + +- zope.interface_: 5.0.0 → 5.1.0 + +(:issue:`5512`, :issue:`5514`, :issue:`5524`, :issue:`5563`, :issue:`5664`, +:issue:`5670`, :issue:`5678`) + + +Deprecations +~~~~~~~~~~~~ + +- :meth:`ImagesPipeline.thumb_path + ` must now accept an + ``item`` parameter (:issue:`5504`, :issue:`5508`). + +- The ``scrapy.downloadermiddlewares.decompression`` module is now + deprecated (:issue:`5546`, :issue:`5547`). + + +New features +~~~~~~~~~~~~ + +- The + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` + method of :ref:`spider middlewares ` can now be + defined as an :term:`asynchronous generator` (:issue:`4978`). + +- The output of :class:`~scrapy.Request` callbacks defined as + :ref:`coroutines ` is now processed asynchronously + (:issue:`4978`). + +- :class:`~scrapy.spiders.crawl.CrawlSpider` now supports :ref:`asynchronous + callbacks ` (:issue:`5657`). + +- New projects created with the :command:`startproject` command have + :ref:`asyncio support ` enabled by default (:issue:`5590`, + :issue:`5679`). + +- The :setting:`FEED_EXPORT_FIELDS` setting can now be defined as a + dictionary to customize the output name of item fields, lifting the + restriction that required output names to be valid Python identifiers, e.g. + preventing them to have whitespace (:issue:`1008`, :issue:`3266`, + :issue:`3696`). + +- You can now customize :ref:`request fingerprinting ` + through the new :setting:`REQUEST_FINGERPRINTER_CLASS` setting, instead of + having to change it on every Scrapy component that relies on request + fingerprinting (:issue:`900`, :issue:`3420`, :issue:`4113`, :issue:`4762`, + :issue:`4524`). + +- ``jsonl`` is now supported and encouraged as a file extension for `JSON + Lines`_ files (:issue:`4848`). + + .. _JSON Lines: https://jsonlines.org/ + +- :meth:`ImagesPipeline.thumb_path + ` now receives the + source :ref:`item ` (:issue:`5504`, :issue:`5508`). + + +Bug fixes +~~~~~~~~~ + +- When using Google Cloud Storage with a :ref:`media pipeline + `, :setting:`FILES_EXPIRES` now also works when + :setting:`FILES_STORE` does not point at the root of your Google Cloud + Storage bucket (:issue:`5317`, :issue:`5318`). + +- The :command:`parse` command now supports :ref:`asynchronous callbacks + ` (:issue:`5424`, :issue:`5577`). + +- When using the :command:`parse` command with a URL for which there is no + available spider, an exception is no longer raised (:issue:`3264`, + :issue:`3265`, :issue:`5375`, :issue:`5376`, :issue:`5497`). + +- :class:`~scrapy.http.TextResponse` now gives higher priority to the `byte + order mark`_ when determining the text encoding of the response body, + following the `HTML living standard`_ (:issue:`5601`, :issue:`5611`). + + .. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark + .. _HTML living standard: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding + +- MIME sniffing takes the response body into account in FTP and HTTP/1.0 + requests, as well as in cached requests (:issue:`4873`). + +- MIME sniffing now detects valid HTML 5 documents even if the ``html`` tag + is missing (:issue:`4873`). + +- An exception is now raised if :setting:`ASYNCIO_EVENT_LOOP` has a value + that does not match the asyncio event loop actually installed + (:issue:`5529`). + +- Fixed :meth:`Headers.getlist ` + returning only the last header (:issue:`5515`, :issue:`5526`). + +- Fixed :class:`LinkExtractor + ` not ignoring the + ``tar.gz`` file extension by default (:issue:`1837`, :issue:`2067`, + :issue:`4066`) + + +Documentation +~~~~~~~~~~~~~ + +- Clarified the return type of :meth:`Spider.parse ` + (:issue:`5602`, :issue:`5608`). + +- To enable + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + to do `brotli compression`_, installing brotli_ is now recommended instead + of installing brotlipy_, as the former provides a more recent version of + brotli. + + .. _brotli: https://github.com/google/brotli + .. _brotli compression: https://www.ietf.org/rfc/rfc7932.txt + +- :ref:`Signal documentation ` now mentions :ref:`coroutine + support ` and uses it in code examples (:issue:`4852`, + :issue:`5358`). + +- :ref:`bans` now recommends `Common Crawl`_ instead of `Google cache`_ + (:issue:`3582`, :issue:`5432`). + + .. _Common Crawl: https://commoncrawl.org/ + .. _Google cache: http://www.googleguide.com/cached_pages.html + +- The new :ref:`topics-components` topic covers enforcing requirements on + Scrapy components, like :ref:`downloader middlewares + `, :ref:`extensions `, + :ref:`item pipelines `, :ref:`spider middlewares + `, and more; :ref:`enforce-asyncio-requirement` + has also been added (:issue:`4978`). + +- :ref:`topics-settings` now indicates that setting values must be + :ref:`picklable ` (:issue:`5607`, :issue:`5629`). + +- Removed outdated documentation (:issue:`5446`, :issue:`5373`, + :issue:`5369`, :issue:`5370`, :issue:`5554`). + +- Fixed typos (:issue:`5442`, :issue:`5455`, :issue:`5457`, :issue:`5461`, + :issue:`5538`, :issue:`5553`, :issue:`5558`, :issue:`5624`, :issue:`5631`). + +- Fixed other issues (:issue:`5283`, :issue:`5284`, :issue:`5559`, + :issue:`5567`, :issue:`5648`, :issue:`5659`, :issue:`5665`). + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added a continuous integration job to run `twine check`_ (:issue:`5655`, + :issue:`5656`). + + .. _twine check: https://twine.readthedocs.io/en/stable/#twine-check + +- Addressed test issues and warnings (:issue:`5560`, :issue:`5561`, + :issue:`5612`, :issue:`5617`, :issue:`5639`, :issue:`5645`, :issue:`5662`, + :issue:`5671`, :issue:`5675`). + +- Cleaned up code (:issue:`4991`, :issue:`4995`, :issue:`5451`, + :issue:`5487`, :issue:`5542`, :issue:`5667`, :issue:`5668`, :issue:`5672`). + +- Applied minor code improvements (:issue:`5661`). + + .. _release-2.6.3: Scrapy 2.6.3 (2022-09-27) @@ -3139,7 +3328,7 @@ New Features ~~~~~~~~~~~~ - Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`) -- Support `brotli`_-compressed content; requires optional `brotlipy`_ +- Support `brotli-compressed`_ content; requires optional `brotlipy`_ (:issue:`2535`) - New :ref:`response.follow ` shortcut for creating requests (:issue:`1940`) @@ -3176,7 +3365,7 @@ New Features - ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command (:issue:`2740`) -.. _brotli: https://github.com/google/brotli +.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://github.com/python-hyper/brotlipy/ Bug fixes diff --git a/docs/topics/components.rst b/docs/topics/components.rst index c44f3def2..ca301b827 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -75,9 +75,9 @@ If your requirement is a minimum Scrapy version, you may use class MyComponent: def __init__(self): - if parse_version(scrapy.__version__) < parse_version('VERSION'): + if parse_version(scrapy.__version__) < parse_version('2.7'): raise RuntimeError( - f"{MyComponent.__qualname__} requires Scrapy VERSION or " + f"{MyComponent.__qualname__} requires Scrapy 2.7 or " f"later, which allow defining the process_spider_output " f"method of spider middlewares as an asynchronous " f"generator." diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 750263385..a1ba4ba5c 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -22,7 +22,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): If you are using any custom or third-party :ref:`spider middleware `, see :ref:`sync-async-spider-middleware`. - .. versionchanged:: VERSION + .. versionchanged:: 2.7 Output of async callbacks is now processed asynchronously instead of collecting all of it first. @@ -49,7 +49,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): See also :ref:`sync-async-spider-middleware` and :ref:`universal-spider-middleware`. - .. versionadded:: VERSION + .. versionadded:: 2.7 General usage ============= @@ -129,7 +129,7 @@ Common use cases for asynchronous code include: Mixing synchronous and asynchronous spider middlewares ====================================================== -.. versionadded:: VERSION +.. versionadded:: 2.7 The output of a :class:`~scrapy.Request` callback is passed as the ``result`` parameter to the @@ -182,10 +182,10 @@ process_spider_output_async method `. Universal spider middlewares ============================ -.. versionadded:: VERSION +.. versionadded:: 2.7 To allow writing a spider middleware that supports asynchronous execution of -its ``process_spider_output`` method in Scrapy VERSION and later (avoiding +its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding :ref:`asynchronous-to-synchronous conversions `) while maintaining support for older Scrapy versions, you may define ``process_spider_output`` as a synchronous method and define an @@ -206,7 +206,7 @@ For example:: yield r .. note:: This is an interim measure to allow, for a time, to write code that - works in Scrapy VERSION and later without requiring + works in Scrapy 2.7 and later without requiring asynchronous-to-synchronous conversions, and works in earlier Scrapy versions as well. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 49cb69f67..4393e1c68 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -394,7 +394,7 @@ To change how request fingerprints are built for your requests, use the REQUEST_FINGERPRINTER_CLASS ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: VERSION +.. versionadded:: 2.7 Default: :class:`scrapy.utils.request.RequestFingerprinter` @@ -409,38 +409,38 @@ import path. REQUEST_FINGERPRINTER_IMPLEMENTATION ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: VERSION +.. versionadded:: 2.7 -Default: ``'PREVIOUS_VERSION'`` +Default: ``'2.6'`` Determines which request fingerprinting algorithm is used by the default request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`). Possible values are: -- ``'PREVIOUS_VERSION'`` (default) +- ``'2.6'`` (default) This implementation uses the same request fingerprinting algorithm as - Scrapy PREVIOUS_VERSION and earlier versions. + Scrapy 2.6 and earlier versions. Even though this is the default value for backward compatibility reasons, it is a deprecated value. -- ``'VERSION'`` +- ``'2.7'`` - This implementation was introduced in Scrapy VERSION to fix an issue of the + This implementation was introduced in Scrapy 2.7 to fix an issue of the previous implementation. New projects should use this value. The :command:`startproject` command sets this value in the generated ``settings.py`` file. -If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are +If you are using the default value (``'2.6'``) for this setting, and you are using Scrapy components where changing the request fingerprinting algorithm would cause undesired results, you need to carefully decide when to change the value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS` -setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request +setting to a custom request fingerprinter class that implements the 2.6 request fingerprinting algorithm and does not log this warning ( -:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a +:ref:`2.6-request-fingerprinter` includes an example implementation of such a class). Scenarios where changing the request fingerprinting algorithm may cause @@ -449,14 +449,14 @@ undesired results include, for example, using the HTTP cache middleware (see Changing the request fingerprinting algorithm would invalidade the current cache, requiring you to redownload all requests again. -Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in +Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in your settings to switch already to the request fingerprinting implementation that will be the only request fingerprinting implementation available in a future version of Scrapy, and remove the deprecation warning triggered by using -the default value (``'PREVIOUS_VERSION'``). +the default value (``'2.6'``). -.. _PREVIOUS_VERSION-request-fingerprinter: +.. _2.6-request-fingerprinter: .. _custom-request-fingerprinter: Writing your own request fingerprinter @@ -464,6 +464,8 @@ Writing your own request fingerprinter A request fingerprinter is a class that must implement the following method: +.. currentmodule:: None + .. method:: fingerprint(self, request) Return a :class:`bytes` object that uniquely identifies *request*. @@ -476,6 +478,7 @@ A request fingerprinter is a class that must implement the following method: Additionally, it may also implement the following methods: .. classmethod:: from_crawler(cls, crawler) + :noindex: If present, this class method is called to create a request fingerprinter instance from a :class:`~scrapy.crawler.Crawler` object. It must return a @@ -495,11 +498,13 @@ Additionally, it may also implement the following methods: :class:`~scrapy.settings.Settings` object. It must return a new instance of the request fingerprinter. -The ``fingerprint`` method of the default request fingerprinter, +.. currentmodule:: scrapy.http + +The :meth:`fingerprint` method of the default request fingerprinter, :class:`scrapy.utils.request.RequestFingerprinter`, uses :func:`scrapy.utils.request.fingerprint` with its default parameters. For some -common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well -in your ``fingerprint`` method implementation: +common use cases you can use :func:`scrapy.utils.request.fingerprint` as well +in your :meth:`fingerprint` method implementation: .. autofunction:: scrapy.utils.request.fingerprint @@ -519,7 +524,7 @@ account:: You can also write your own fingerprinting logic from scratch. -However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure +However, if you do not use :func:`scrapy.utils.request.fingerprint`, make sure you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints: - Caching saves CPU by ensuring that fingerprints are calculated only once @@ -553,7 +558,7 @@ If you need to be able to override the request fingerprinting for arbitrary requests from your spider callbacks, you may implement a request fingerprinter that reads fingerprints from :attr:`request.meta ` when available, and then falls back to -:func:`~scrapy.utils.request.fingerprint`. For example:: +:func:`scrapy.utils.request.fingerprint`. For example:: from scrapy.utils.request import fingerprint @@ -564,8 +569,8 @@ when available, and then falls back to return request.meta['fingerprint'] return fingerprint(request) -If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION -without using the deprecated ``'PREVIOUS_VERSION'`` value of the +If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6 +without using the deprecated ``'2.6'`` value of the :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following request fingerprinter:: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0b1ef71cf..40bcda288 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1642,7 +1642,7 @@ install the default reactor defined by Twisted for the current platform. This is to maintain backward compatibility and avoid possible problems caused by using a non-default reactor. -.. versionchanged:: VERSION +.. versionchanged:: 2.7 The :command:`startproject` command now sets this setting to ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated ``settings.py`` file. @@ -1661,14 +1661,14 @@ Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. -This setting can act as a stopping condition in case of URLs of ever-increasing -length, which may be caused for example by a programming error either in the -target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and +This setting can act as a stopping condition in case of URLs of ever-increasing +length, which may be caused for example by a programming error either in the +target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and :setting:`DEPTH_LIMIT`. Use ``0`` to allow URLs of any length. -The default value is copied from the `Microsoft Internet Explorer maximum URL +The default value is copied from the `Microsoft Internet Explorer maximum URL length`_, even though this setting exists for different reasons. .. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 816cb5e03..303401a3c 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -105,17 +105,17 @@ object gives you access, for example, to the :ref:`settings `. :class:`~scrapy.Request` objects and :ref:`item objects `. - .. versionchanged:: VERSION + .. versionchanged:: 2.7 This method may be defined as an :term:`asynchronous generator`, in which case ``result`` is an :term:`asynchronous iterable`. Consider defining this method as an :term:`asynchronous generator`, which will be a requirement in a future version of Scrapy. However, if you plan on sharing your spider middleware with other people, consider - either :ref:`enforcing Scrapy VERSION ` + either :ref:`enforcing Scrapy 2.7 ` as a minimum requirement of your spider middleware, or :ref:`making your spider middleware universal ` so that - it works with Scrapy versions earlier than Scrapy VERSION. + it works with Scrapy versions earlier than Scrapy 2.7. :param response: the response which generated this output from the spider @@ -130,7 +130,7 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_spider_output_async(response, result, spider) - .. versionadded:: VERSION + .. versionadded:: 2.7 If defined, this method must be an :term:`asynchronous generator`, which will be called instead of :meth:`process_spider_output` if diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ff86af125..29ff028be 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -248,7 +248,7 @@ REFERER_ENABLED = True REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter' -REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION' +REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.6' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index c0c34e986..bbf60982c 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -88,5 +88,5 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' # Set settings whose default value is deprecated to a future-proof value -REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION' +REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7' TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index cf33317ce..fbddc41fb 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -236,10 +236,10 @@ class RequestFingerprinter: 'REQUEST_FINGERPRINTER_IMPLEMENTATION' ) else: - implementation = 'PREVIOUS_VERSION' - if implementation == 'PREVIOUS_VERSION': + implementation = '2.6' + if implementation == '2.6': message = ( - '\'PREVIOUS_VERSION\' is a deprecated value for the ' + '\'2.6\' is a deprecated value for the ' '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n' '\n' 'It is also the default value. In other words, it is normal ' @@ -254,14 +254,14 @@ class RequestFingerprinter: ) warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) self._fingerprint = _request_fingerprint_as_bytes - elif implementation == 'VERSION': + elif implementation == '2.7': self._fingerprint = fingerprint else: raise ValueError( f'Got an invalid value on setting ' f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': ' - f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) ' - f'and \'VERSION\'.' + f'{implementation!r}. Valid values are \'2.6\' (deprecated) ' + f'and \'2.7\'.' ) def fingerprint(self, request): diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 0b828f7c0..445cd2e3a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -65,7 +65,7 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): # Set by default settings that prevent deprecation warnings. settings = {} if prevent_warnings: - settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION' + settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = '2.7' settings.update(settings_dict or {}) runner = CrawlerRunner(settings) return runner.create_crawler(spidercls or Spider) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 8be4b6fe1..5383ec652 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -350,7 +350,7 @@ with multiples lines @defer.inlineCallbacks def test_crawl_multiple(self): - runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}) + runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}) runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c61d461f7..da6024c2b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -110,7 +110,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): 'LOG_LEVEL': 'INFO', 'LOG_FILE': log_file, # settings to avoid extra warnings - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, } @@ -235,7 +235,7 @@ class NoRequestsSpider(scrapy.Spider): class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): - return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}) + return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}) @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful(self): @@ -283,14 +283,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", }) else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", }) yield runner.crawl(NoRequestsSpider) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 8a37a8ebe..6ebb716b0 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -51,7 +51,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_crawler_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -60,7 +60,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_settings_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -68,7 +68,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_direct_scheduler(self): settings = {'DUPEFILTER_CLASS': DirectDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, 'n/a') @@ -172,7 +172,7 @@ class RFPDupeFilterTest(unittest.TestCase): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) @@ -199,7 +199,7 @@ class RFPDupeFilterTest(unittest.TestCase): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) @@ -233,7 +233,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug_default_dupefilter(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index e46532a1c..0e174cd34 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -64,7 +64,7 @@ class FileDownloadCrawlTestCase(TestCase): self.tmpmediastore = self.mktemp() os.mkdir(self.tmpmediastore) self.settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', 'ITEM_PIPELINES': {self.pipeline_class: 1}, self.store_setting_key: self.tmpmediastore, } diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index ac66056ba..50a7755c1 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -52,7 +52,7 @@ class MockCrawler(Crawler): SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, JOBDIR=jobdir, DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', - REQUEST_FINGERPRINTER_IMPLEMENTATION='VERSION', + REQUEST_FINGERPRINTER_IMPLEMENTATION='2.7', ) super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 3719c7c9f..7a590f96c 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -98,7 +98,7 @@ class SpiderLoaderTest(unittest.TestCase): module = 'tests.test_spiderloader.test_spiders.spider1' runner = CrawlerRunner({ 'SPIDER_MODULES': [module], - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', }) self.assertRaisesRegex(KeyError, 'Spider not found', diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 8bc7922b6..a92d9a0ac 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -505,7 +505,7 @@ class RequestFingerprinterTestCase(unittest.TestCase): def test_deprecated_implementation(self): settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.6', } with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) @@ -518,7 +518,7 @@ class RequestFingerprinterTestCase(unittest.TestCase): def test_recommended_implementation(self): settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', } with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) From 20b79a0f2e47800bf4648c7f890c8170fc8f5ede Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Oct 2022 19:09:22 +0600 Subject: [PATCH 46/51] =?UTF-8?q?Bump=20version:=202.6.2=20=E2=86=92=202.7?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2e2f7949a..f88071685 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.6.2 +current_version = 2.7.0 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index d8b9fcd1e..1ec183a1d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.7.0: -Scrapy 2.7.0 (to be released) +Scrapy 2.7.0 (2022-10-17) ----------------------------- Highlights: diff --git a/scrapy/VERSION b/scrapy/VERSION index 097a15a2a..24ba9a38d 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.6.2 +2.7.0 From b33244e2f0d877b8911f949308222db0b076d665 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 21 Oct 2022 19:17:04 +0500 Subject: [PATCH 47/51] Fix the flake8 per-file ignore syntax (#5688) --- .flake8 | 17 +++++++++-------- scrapy/utils/url.py | 1 + tests/test_loader.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.flake8 b/.flake8 index d7aebc24b..0c64d009e 100644 --- a/.flake8 +++ b/.flake8 @@ -6,16 +6,17 @@ ignore = W503 exclude = docs/conf.py +per-file-ignores = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file # F401: Module imported but unused - scrapy/__init__.py E402 - scrapy/core/downloader/handlers/http.py F401 - scrapy/http/__init__.py F401 - scrapy/linkextractors/__init__.py E402 F401 - scrapy/selector/__init__.py F401 - scrapy/spiders/__init__.py E402 F401 + scrapy/__init__.py:E402 + scrapy/core/downloader/handlers/http.py:F401 + scrapy/http/__init__.py:F401 + scrapy/linkextractors/__init__.py:E402,F401 + scrapy/selector/__init__.py:F401 + scrapy/spiders/__init__.py:E402,F401 # Issues pending a review: - scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 + scrapy/utils/url.py:F403,F405 + tests/test_loader.py:E741 diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 4d5e9ae82..21201ace5 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -34,6 +34,7 @@ def url_has_any_extension(url, extensions): lowercase_path = parse_url(url).path.lower() return any(lowercase_path.endswith(ext) for ext in extensions) + def parse_url(url, encoding=None): """Return urlparsed url from the given argument (which could be an already parsed url) diff --git a/tests/test_loader.py b/tests/test_loader.py index c0937b349..b3e44d36b 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -295,7 +295,7 @@ class SelectortemLoaderTest(unittest.TestCase): l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), ['Marta']) - + def test_init_method_with_base_response(self): """Selector should be None after initialization""" response = Response("https://scrapy.org") From b61b71c6f015d45f6e98a4280bf4993517180045 Mon Sep 17 00:00:00 2001 From: Godson-Gnanaraj Date: Tue, 25 Oct 2022 08:44:43 +0530 Subject: [PATCH 48/51] Replace indentation of source before parsing with ast. closes #5323 --- scrapy/utils/misc.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 1221b39b2..c0258c8d9 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -226,7 +226,14 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - code = re.sub(r"^[\t ]+", "", inspect.getsource(callable)) + pattern = r"(^[\t ]+)" + src = inspect.getsource(callable) + match = re.match(pattern, src) # Find indentation + code = re.sub(pattern, "", src) + if match: + # Remove indentation + code = re.sub(f"\n{match.group(0)}", "\n", code) + tree = ast.parse(code) for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): From f4e2a10ed6a44300738bd3701ea62b432c9c1a06 Mon Sep 17 00:00:00 2001 From: Kaushal Sharma Date: Tue, 25 Oct 2022 02:45:46 -0700 Subject: [PATCH 49/51] =?UTF-8?q?Image.ANTIALIAS=20=E2=86=92=20Image.Resam?= =?UTF-8?q?pling.LANCZOS=20(#5692)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/pipelines/images.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 6b97190ee..67b3224b3 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -160,7 +160,14 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() - image.thumbnail(size, self._Image.ANTIALIAS) + try: + # Image.Resampling.LANCZOS was added in Pillow 9.1.0 + # remove this try except block, + # when updating the minimum requirements for Pillow. + resampling_filter = self._Image.Resampling.LANCZOS + except AttributeError: + resampling_filter = self._Image.ANTIALIAS + image.thumbnail(size, resampling_filter) buf = BytesIO() image.save(buf, 'JPEG') From 830e1c5dd85618a27749bbe41d35b11fb2bdd348 Mon Sep 17 00:00:00 2001 From: Godson-Gnanaraj Date: Wed, 26 Oct 2022 01:26:54 +0530 Subject: [PATCH 50/51] Add test for parsing decorated methods --- ...t_return_with_argument_inside_generator.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 1c85ca353..72277d701 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -165,6 +165,89 @@ https://example.org warn_on_generator_with_return_value(None, l2) self.assertEqual(len(w), 0) + def test_generators_return_none_with_decorator(self): + def decorator(func): + def inner_func(): + func() + return inner_func + + @decorator + def f3(): + yield 1 + return None + + @decorator + def g3(): + yield 1 + return + + @decorator + def h3(): + yield 1 + + @decorator + def i3(): + yield 1 + yield from generator_that_returns_stuff() + + @decorator + def j3(): + yield 1 + + def helper(): + return 0 + + yield helper() + + @decorator + def k3(): + """ +docstring + """ + url = """ +https://example.org + """ + yield url + return + + @decorator + def l3(): + return + + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f3) + assert not is_generator_with_return_value(g3) + assert not is_generator_with_return_value(h3) + assert not is_generator_with_return_value(i3) + assert not is_generator_with_return_value(j3) # not recursive + assert not is_generator_with_return_value(k3) # not recursive + assert not is_generator_with_return_value(l3) + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_none) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, f3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, g3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, h3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, i3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, j3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, k3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, l3) + self.assertEqual(len(w), 0) + @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error) def test_indentation_error(self): with warnings.catch_warnings(record=True) as w: From b0ddffc47b9cee5e6146497b42de3787da76d2ad Mon Sep 17 00:00:00 2001 From: Godson-Gnanaraj Date: Wed, 26 Oct 2022 06:53:43 +0530 Subject: [PATCH 51/51] Misc. changes: - compile regex - readability improvements --- scrapy/utils/misc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index c0258c8d9..4d4fb9600 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -226,13 +226,13 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - pattern = r"(^[\t ]+)" src = inspect.getsource(callable) - match = re.match(pattern, src) # Find indentation - code = re.sub(pattern, "", src) + pattern = re.compile(r"(^[\t ]+)") + code = pattern.sub("", src) + + match = pattern.match(src) # finds indentation if match: - # Remove indentation - code = re.sub(f"\n{match.group(0)}", "\n", code) + code = re.sub(f"\n{match.group(0)}", "\n", code) # remove indentation tree = ast.parse(code) for node in walk_callable(tree):