From c4d7f5e7a996e7e5e5c32e947c026d29df5315d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 2 Jan 2023 13:41:08 +0100 Subject: [PATCH 1/7] Use CLang to build Reppy --- .github/workflows/checks.yml | 12 ++++++++++-- .github/workflows/tests-ubuntu.yml | 13 ++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8c1ae4bd3..e5b0306b3 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -14,10 +14,12 @@ jobs: - python-version: "3.11" env: TOXENV: flake8 - # Pylint requires installing reppy, which does not support Python 3.9 - # https://github.com/seomoz/reppy/issues/122 + # Pylint requires installing reppy, which: + # - Does not support Python 3.9: https://github.com/seomoz/reppy/issues/122 + # - Requires CLang to build https://github.com/seomoz/reppy/issues/132 - python-version: 3.8 env: + CC: clang TOXENV: pylint - python-version: 3.7 env: @@ -37,6 +39,12 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Set up Clang + uses: egor-tensin/setup-clang@v1 + with: + version: latest + platform: x64 + - name: Run check env: ${{ matrix.env }} run: | diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 9c3ce8115..4ad722ad5 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -38,11 +38,12 @@ jobs: env: TOXENV: pypy3-pinned - # extras - # extra-deps includes reppy, which does not support Python 3.9 - # https://github.com/seomoz/reppy/issues/122 + # extra-deps includes reppy, which: + # - Does not support Python 3.9: https://github.com/seomoz/reppy/issues/122 + # - Requires CLang to build https://github.com/seomoz/reppy/issues/132 - python-version: 3.8 env: + CC: clang TOXENV: extra-deps steps: @@ -59,6 +60,12 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev + - name: Set up Clang + uses: egor-tensin/setup-clang@v1 + with: + version: latest + platform: x64 + - name: Run tests env: ${{ matrix.env }} run: | From e47ada2c7cc7c31f31e54652d5b7f893678198af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 2 Jan 2023 13:49:25 +0100 Subject: [PATCH 2/7] Add CC to tox.ini:[testenv]passenv --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 0e156d63b..865ef5383 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,7 @@ deps = # Extras botocore>=1.4.87 passenv = + CC S3_TEST_FILE_URI AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY From 6084e8f6270acd0a8e7a0081ab0b60dfbdf56933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 2 Jan 2023 14:00:37 +0100 Subject: [PATCH 3/7] Do not use -U for pip in Tox --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 0e156d63b..bf646242b 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,7 @@ download = true commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = - pip install -U -ctests/upper-constraints.txt {opts} {packages} + python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} [testenv:typing] basepython = python3 @@ -107,7 +107,7 @@ deps = setenv = _SCRAPY_PINNED=true install_command = - pip install -U {opts} {packages} + python -I -m pip install {opts} {packages} [testenv:pinned] deps = From 016d1de64eebf6abd929ab7fb0037334da66677b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 2 Jan 2023 14:13:04 +0100 Subject: [PATCH 4/7] Remove Reppy from CI --- .github/workflows/checks.yml | 12 +----------- .github/workflows/tests-ubuntu.yml | 12 +----------- tox.ini | 8 ++------ 3 files changed, 4 insertions(+), 28 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e5b0306b3..e9f9a6aea 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -14,12 +14,8 @@ jobs: - python-version: "3.11" env: TOXENV: flake8 - # Pylint requires installing reppy, which: - # - Does not support Python 3.9: https://github.com/seomoz/reppy/issues/122 - # - Requires CLang to build https://github.com/seomoz/reppy/issues/132 - - python-version: 3.8 + - python-version: "3.11" env: - CC: clang TOXENV: pylint - python-version: 3.7 env: @@ -39,12 +35,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Set up Clang - uses: egor-tensin/setup-clang@v1 - with: - version: latest - platform: x64 - - name: Run check env: ${{ matrix.env }} run: | diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 4ad722ad5..8fcf90a18 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -38,12 +38,8 @@ jobs: env: TOXENV: pypy3-pinned - # extra-deps includes reppy, which: - # - Does not support Python 3.9: https://github.com/seomoz/reppy/issues/122 - # - Requires CLang to build https://github.com/seomoz/reppy/issues/132 - - python-version: 3.8 + - python-version: "3.11" env: - CC: clang TOXENV: extra-deps steps: @@ -60,12 +56,6 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev - - name: Set up Clang - uses: egor-tensin/setup-clang@v1 - with: - version: latest - platform: x64 - - name: Run tests env: ${{ matrix.env }} run: | diff --git a/tox.ini b/tox.ini index 865ef5383..cc529fc70 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,6 @@ deps = # Extras botocore>=1.4.87 passenv = - CC S3_TEST_FILE_URI AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY @@ -64,8 +63,7 @@ commands = flake8 {posargs:docs scrapy tests} [testenv:pylint] -# reppy does not support Python 3.9+ -basepython = python3.8 +basepython = python3 deps = {[testenv:extra-deps]deps} pylint==2.15.6 @@ -128,8 +126,7 @@ setenv = {[pinned]setenv} [testenv:extra-deps] -# reppy does not support Python 3.9+ -basepython = python3.8 +basepython = python3 deps = {[testenv]deps} boto @@ -137,7 +134,6 @@ deps = # Twisted[http2] currently forces old mitmproxy because of h2 version # restrictions in their deps, so we need to pin old markupsafe here too. markupsafe < 2.1.0 - reppy robotexclusionrulesparser Pillow>=4.0.0 Twisted[http2]>=17.9.0 From fd6742e8117eb76f4a717b33b24790c1419bd838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 2 Jan 2023 19:13:31 +0100 Subject: [PATCH 5/7] Fix tests for pygments 2.14 (#5783) --- tests/test_utils_display.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index 9ec8311d9..1d1fb50e6 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -7,17 +7,27 @@ from scrapy.utils.display import pformat, pprint class TestDisplay(TestCase): object = {'a': 1} - colorized_string = ( - "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" - "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n" - ) + colorized_strings = { + ( + ( + "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" + "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}" + ) + + suffix + ) + for suffix in ( + # https://github.com/pygments/pygments/issues/2313 + "\n", # pygments ≤ 2.13 + "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14 + ) + } plain_string = "{'a': 1}" @mock.patch('sys.platform', 'linux') @mock.patch("sys.stdout.isatty") def test_pformat(self, isatty): isatty.return_value = True - self.assertEqual(pformat(self.object), self.colorized_string) + self.assertIn(pformat(self.object), self.colorized_strings) @mock.patch("sys.stdout.isatty") def test_pformat_dont_colorize(self, isatty): @@ -33,7 +43,7 @@ class TestDisplay(TestCase): def test_pformat_old_windows(self, isatty, version): isatty.return_value = True version.return_value = '10.0.14392' - self.assertEqual(pformat(self.object), self.colorized_string) + self.assertIn(pformat(self.object), self.colorized_strings) @mock.patch('sys.platform', 'win32') @mock.patch('scrapy.utils.display._enable_windows_terminal_processing') @@ -53,7 +63,7 @@ class TestDisplay(TestCase): isatty.return_value = True version.return_value = '10.0.14393' terminal_processing.return_value = True - self.assertEqual(pformat(self.object), self.colorized_string) + self.assertIn(pformat(self.object), self.colorized_strings) @mock.patch('sys.platform', 'linux') @mock.patch("sys.stdout.isatty") From 724b0332878b5df502dc9504aec69c47fb1ea0bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Jan 2023 19:51:07 +0100 Subject: [PATCH 6/7] Fix CI issues related to asyncio (#5782) --- scrapy/utils/defer.py | 7 +++---- scrapy/utils/reactor.py | 28 ++++++++++++++++++++++++---- tests/test_utils_asyncio.py | 3 +++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index ddacfaa49..0da2e3638 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -26,7 +26,7 @@ from twisted.python import failure from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest -from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy +from scrapy.utils.reactor import is_asyncio_reactor_installed, _get_asyncio_event_loop def defer_fail(_failure: Failure) -> Deferred: @@ -267,7 +267,7 @@ def deferred_from_coro(o) -> Any: # that use asyncio, e.g. "await asyncio.sleep(1)" return ensureDeferred(o) # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - event_loop = get_asyncio_event_loop_policy().get_event_loop() + event_loop = _get_asyncio_event_loop() return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) return o @@ -318,8 +318,7 @@ def deferred_to_future(d: Deferred) -> Future: d = treq.get('https://example.com/additional') additional_response = await deferred_to_future(d) """ - policy = get_asyncio_event_loop_policy() - return d.asFuture(policy.get_event_loop()) + return d.asFuture(_get_asyncio_event_loop()) def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index e6b8de292..6a051d8b0 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,6 +1,7 @@ import asyncio import sys from contextlib import suppress +from warnings import catch_warnings, filterwarnings from twisted.internet import asyncioreactor, error @@ -81,6 +82,10 @@ def install_reactor(reactor_path, event_loop_path=None): installer() +def _get_asyncio_event_loop(): + return set_asyncio_event_loop(None) + + def set_asyncio_event_loop(event_loop_path): """Sets and returns the event loop with specified import path.""" policy = get_asyncio_event_loop_policy() @@ -90,11 +95,26 @@ def set_asyncio_event_loop(event_loop_path): asyncio.set_event_loop(event_loop) else: try: - event_loop = policy.get_event_loop() + with catch_warnings(): + # In Python 3.10.9, 3.11.1, 3.12 and 3.13, a DeprecationWarning + # is emitted about the lack of a current event loop, because in + # Python 3.14 and later `get_event_loop` will raise a + # RuntimeError in that event. Because our code is already + # prepared for that future behavior, we ignore the deprecation + # warning. + filterwarnings( + "ignore", + message="There is no current event loop", + category=DeprecationWarning, + ) + event_loop = policy.get_event_loop() except RuntimeError: - # `get_event_loop` is expected to fail when called from a new thread - # with no asyncio event loop yet installed. Such is the case when - # called from `scrapy shell` + # `get_event_loop` raises RuntimeError when called with no asyncio + # event loop yet installed in the following scenarios: + # - From a thread other than the main thread. For example, when + # using ``scrapy shell``. + # - Previsibly on Python 3.14 and later. + # https://github.com/python/cpython/issues/100160#issuecomment-1345581902 event_loop = policy.new_event_loop() asyncio.set_event_loop(event_loop) return event_loop diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 741c6a505..42780ace7 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -14,6 +14,9 @@ class AsyncioTest(TestCase): self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') def test_install_asyncio_reactor(self): + from twisted.internet import reactor as original_reactor with warnings.catch_warnings(record=True) as w: install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") self.assertEqual(len(w), 0) + from twisted.internet import reactor + assert original_reactor == reactor From 16546564f673b0d58c55584870149424abc73fee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Jan 2023 20:16:54 +0100 Subject: [PATCH 7/7] =?UTF-8?q?Minimum=20cryptography:=203.3=20=E2=86=92?= =?UTF-8?q?=203.4.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 27445cad0..bdae28047 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support(): install_requires = [ 'Twisted>=18.9.0', - 'cryptography>=3.3', + 'cryptography>=3.4.6', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', diff --git a/tox.ini b/tox.ini index 84e83ac2a..520d90303 100644 --- a/tox.ini +++ b/tox.ini @@ -81,7 +81,7 @@ commands = [pinned] deps = - cryptography==3.3 + cryptography==3.4.6 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0