Merge pull request #5221 from wRAR/3.10-tests

Fixing tests for upcoming 3.10.

.github/workflows/tests-ubuntu.yml

@@ -41,7 +41,12 @@ jobs:
           env:
             TOXENV: extra-deps
             TOX_PIP_VERSION: 20.3.3
-        - python-version: 3.9
+
+        # 3.10-pre
+        - python-version: "3.10.0-beta.4"
+          env:
+            TOXENV: py
+        - python-version: "3.10.0-beta.4"
           env:
             TOXENV: asyncio
 
@@ -54,7 +59,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install system libraries
-      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')
+      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4'
       run: |
         sudo apt-get update
         # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version

scrapy/utils/signal.py

@@ -1,5 +1,5 @@
 """Helper functions for working with signals"""
-import collections
+import collections.abc
 import logging
 
 from twisted.internet.defer import DeferredList, Deferred
@@ -21,7 +21,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     Failures instead of exceptions.
     """
     dont_log = named.pop('dont_log', ())
-    dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,)
+    dont_log = tuple(dont_log) if isinstance(dont_log, collections.abc.Sequence) else (dont_log,)
     dont_log += (StopDownload, )
     spider = named.get('spider', None)
     responses = []

tests/test_http_response.py

@@ -878,4 +878,4 @@ class CustomResponseTest(TextResponseTest):
 
         with self.assertRaises(TypeError) as ctx:
             r1.replace(unknown="unknown")
-        self.assertEqual(str(ctx.exception), "__init__() got an unexpected keyword argument 'unknown'")
+        self.assertTrue(str(ctx.exception).endswith("__init__() got an unexpected keyword argument 'unknown'"))

tests/test_request_cb_kwargs.py

@@ -158,12 +158,14 @@ class CallbackKeywordArgumentsTestCase(TestCase):
                 if key in line.getMessage():
                     exceptions[key] = line
         self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
-        self.assertEqual(
-            str(exceptions['takes_less'].exc_info[1]),
-            "parse_takes_less() got an unexpected keyword argument 'number'"
+        self.assertTrue(
+            str(exceptions['takes_less'].exc_info[1]).endswith(
+                "parse_takes_less() got an unexpected keyword argument 'number'"
+            )
         )
         self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
-        self.assertEqual(
-            str(exceptions['takes_more'].exc_info[1]),
-            "parse_takes_more() missing 1 required positional argument: 'other'"
+        self.assertTrue(
+            str(exceptions['takes_more'].exc_info[1]).endswith(
+                "parse_takes_more() missing 1 required positional argument: 'other'"
+            )
         )

tests/test_spiderloader/__init__.py

@@ -118,6 +118,11 @@ class SpiderLoaderTest(unittest.TestCase):
             settings = Settings({'SPIDER_MODULES': [module],
                                  'SPIDER_LOADER_WARN_ONLY': True})
             spider_loader = SpiderLoader.from_settings(settings)
+            if str(w[0].message).startswith("_SixMetaPathImporter"):
+                # needed on 3.10 because of https://github.com/benjaminp/six/issues/349,
+                # at least until all six versions we can import (including botocore.vendored.six)
+                # are updated to 1.16.0+
+                w.pop(0)
             self.assertIn("Could not load spiders from module", str(w[0].message))
 
             spiders = spider_loader.list()