From 2bf2f9d6db89968bcb5df6bc7d093fdd53de5ba5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Aug 2021 19:44:11 +0500 Subject: [PATCH 1/5] Add Python 3.10b4 tests on Ubuntu. --- .github/workflows/tests-ubuntu.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 57188bd63..57c994158 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -45,6 +45,14 @@ jobs: env: TOXENV: asyncio + # 3.10-pre + - python-version: "3.10.0-beta.4" + env: + TOXENV: py + - python-version: "3.10.0-beta.4" + env: + TOXENV: asyncio + steps: - uses: actions/checkout@v2 @@ -54,7 +62,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4' run: | sudo apt-get update # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version From ef6fb933b568c497ab3745284bc2a02725bced1c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 20 Jul 2021 12:02:15 +0500 Subject: [PATCH 2/5] Fix a Python 3.10 logging issue. --- scrapy/utils/signal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 62808f3ce..fbafc9d45 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,5 +1,5 @@ """Helper functions for working with signals""" -import collections +import collections.abc import logging from twisted.internet.defer import DeferredList, Deferred @@ -21,7 +21,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): Failures instead of exceptions. """ dont_log = named.pop('dont_log', ()) - dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,) + dont_log = tuple(dont_log) if isinstance(dont_log, collections.abc.Sequence) else (dont_log,) dont_log += (StopDownload, ) spider = named.get('spider', None) responses = [] From 93bf1ae7e3966d56539411c59d172c2971ee61e0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Aug 2021 20:16:29 +0500 Subject: [PATCH 3/5] Fix tests for the 3.10 TypeError message change. --- tests/test_http_response.py | 2 +- tests/test_request_cb_kwargs.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index cf34a9e5c..c376a46cd 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -878,4 +878,4 @@ class CustomResponseTest(TextResponseTest): with self.assertRaises(TypeError) as ctx: r1.replace(unknown="unknown") - self.assertEqual(str(ctx.exception), "__init__() got an unexpected keyword argument 'unknown'") + self.assertTrue(str(ctx.exception).endswith("__init__() got an unexpected keyword argument 'unknown'")) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 145a4e9b2..b68184b87 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -158,12 +158,14 @@ class CallbackKeywordArgumentsTestCase(TestCase): if key in line.getMessage(): exceptions[key] = line self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError) - self.assertEqual( - str(exceptions['takes_less'].exc_info[1]), - "parse_takes_less() got an unexpected keyword argument 'number'" + self.assertTrue( + str(exceptions['takes_less'].exc_info[1]).endswith( + "parse_takes_less() got an unexpected keyword argument 'number'" + ) ) self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) - self.assertEqual( - str(exceptions['takes_more'].exc_info[1]), - "parse_takes_more() missing 1 required positional argument: 'other'" + self.assertTrue( + str(exceptions['takes_more'].exc_info[1]).endswith( + "parse_takes_more() missing 1 required positional argument: 'other'" + ) ) From 983b89ad4f72730c37b32c9240a697a4c1f24183 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 11 Aug 2021 10:39:23 +0500 Subject: [PATCH 4/5] Fix SpiderLoaderTest on Python 3.10. --- tests/test_spiderloader/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 4929f1e3e..8a35e9fd7 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -118,6 +118,11 @@ class SpiderLoaderTest(unittest.TestCase): settings = Settings({'SPIDER_MODULES': [module], 'SPIDER_LOADER_WARN_ONLY': True}) spider_loader = SpiderLoader.from_settings(settings) + if str(w[0].message).startswith("_SixMetaPathImporter"): + # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, + # at least until all six versions we can import (including botocore.vendored.six) + # are updated to 1.16.0+ + w.pop(0) self.assertIn("Could not load spiders from module", str(w[0].message)) spiders = spider_loader.list() From 74cee38a4e07c31a8dd2a8772ff18d1b9adf8a6b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 11 Aug 2021 14:19:08 +0500 Subject: [PATCH 5/5] Don't run the asyncio tests on 3.9. --- .github/workflows/tests-ubuntu.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 57c994158..81beda5da 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -41,9 +41,6 @@ jobs: env: TOXENV: extra-deps TOX_PIP_VERSION: 20.3.3 - - python-version: 3.9 - env: - TOXENV: asyncio # 3.10-pre - python-version: "3.10.0-beta.4"