Merge remote-tracking branch 'upstream/master' into integrate-mime

.github/workflows/checks.yml

@@ -14,9 +14,7 @@ jobs:
         - python-version: "3.11"
           env:
             TOXENV: flake8
-        # Pylint requires installing reppy, which does not support Python 3.9
-        # https://github.com/seomoz/reppy/issues/122
-        - python-version: 3.8
+        - python-version: "3.11"
           env:
             TOXENV: pylint
         - python-version: 3.7

.github/workflows/tests-ubuntu.yml

@@ -38,10 +38,7 @@ jobs:
           env:
             TOXENV: pypy3-pinned
 
-        # extras
-        # extra-deps includes reppy, which does not support Python 3.9
-        # https://github.com/seomoz/reppy/issues/122
-        - python-version: 3.8
+        - python-version: "3.11"
           env:
             TOXENV: extra-deps
 

scrapy/utils/defer.py

@@ -26,7 +26,7 @@ from twisted.python import failure
 from twisted.python.failure import Failure
 
 from scrapy.exceptions import IgnoreRequest
-from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy
+from scrapy.utils.reactor import is_asyncio_reactor_installed, _get_asyncio_event_loop
 
 
 def defer_fail(_failure: Failure) -> Deferred:
@@ -267,7 +267,7 @@ def deferred_from_coro(o) -> Any:
             # that use asyncio, e.g. "await asyncio.sleep(1)"
             return ensureDeferred(o)
         # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
-        event_loop = get_asyncio_event_loop_policy().get_event_loop()
+        event_loop = _get_asyncio_event_loop()
         return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))
     return o
 
@@ -318,8 +318,7 @@ def deferred_to_future(d: Deferred) -> Future:
                 d = treq.get('https://example.com/additional')
                 additional_response = await deferred_to_future(d)
     """
-    policy = get_asyncio_event_loop_policy()
-    return d.asFuture(policy.get_event_loop())
+    return d.asFuture(_get_asyncio_event_loop())
 
 
 def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:

scrapy/utils/reactor.py

@@ -1,6 +1,7 @@
 import asyncio
 import sys
 from contextlib import suppress
+from warnings import catch_warnings, filterwarnings
 
 from twisted.internet import asyncioreactor, error
 
@@ -81,6 +82,10 @@ def install_reactor(reactor_path, event_loop_path=None):
             installer()
 
 
+def _get_asyncio_event_loop():
+    return set_asyncio_event_loop(None)
+
+
 def set_asyncio_event_loop(event_loop_path):
     """Sets and returns the event loop with specified import path."""
     policy = get_asyncio_event_loop_policy()
@@ -90,11 +95,26 @@ def set_asyncio_event_loop(event_loop_path):
         asyncio.set_event_loop(event_loop)
     else:
         try:
-            event_loop = policy.get_event_loop()
+            with catch_warnings():
+                # In Python 3.10.9, 3.11.1, 3.12 and 3.13, a DeprecationWarning
+                # is emitted about the lack of a current event loop, because in
+                # Python 3.14 and later `get_event_loop` will raise a
+                # RuntimeError in that event. Because our code is already
+                # prepared for that future behavior, we ignore the deprecation
+                # warning.
+                filterwarnings(
+                    "ignore",
+                    message="There is no current event loop",
+                    category=DeprecationWarning,
+                )
+                event_loop = policy.get_event_loop()
         except RuntimeError:
-            # `get_event_loop` is expected to fail when called from a new thread
-            # with no asyncio event loop yet installed. Such is the case when
-            # called from `scrapy shell`
+            # `get_event_loop` raises RuntimeError when called with no asyncio
+            # event loop yet installed in the following scenarios:
+            # - From a thread other than the main thread. For example, when
+            #   using ``scrapy shell``.
+            # - Previsibly on Python 3.14 and later.
+            #   https://github.com/python/cpython/issues/100160#issuecomment-1345581902
             event_loop = policy.new_event_loop()
             asyncio.set_event_loop(event_loop)
     return event_loop

setup.py

@@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support():
 
 install_requires = [
     'Twisted>=18.9.0',
-    'cryptography>=3.3',
+    'cryptography>=3.4.6',
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
     'parsel>=1.5.0',

tests/test_utils_asyncio.py

@@ -14,6 +14,9 @@ class AsyncioTest(TestCase):
         self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
 
     def test_install_asyncio_reactor(self):
+        from twisted.internet import reactor as original_reactor
         with warnings.catch_warnings(record=True) as w:
             install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
             self.assertEqual(len(w), 0)
+        from twisted.internet import reactor
+        assert original_reactor == reactor

tests/test_utils_display.py

@@ -7,17 +7,27 @@ from scrapy.utils.display import pformat, pprint
 
 class TestDisplay(TestCase):
     object = {'a': 1}
-    colorized_string = (
-        "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
-        "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n"
-    )
+    colorized_strings = {
+        (
+            (
+                "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
+                "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}"
+            )
+            + suffix
+        )
+        for suffix in (
+            # https://github.com/pygments/pygments/issues/2313
+            "\n",  # pygments ≤ 2.13
+            "\x1b[37m\x1b[39;49;00m\n",  # pygments ≥ 2.14
+        )
+    }
     plain_string = "{'a': 1}"
 
     @mock.patch('sys.platform', 'linux')
     @mock.patch("sys.stdout.isatty")
     def test_pformat(self, isatty):
         isatty.return_value = True
-        self.assertEqual(pformat(self.object), self.colorized_string)
+        self.assertIn(pformat(self.object), self.colorized_strings)
 
     @mock.patch("sys.stdout.isatty")
     def test_pformat_dont_colorize(self, isatty):
@@ -33,7 +43,7 @@ class TestDisplay(TestCase):
     def test_pformat_old_windows(self, isatty, version):
         isatty.return_value = True
         version.return_value = '10.0.14392'
-        self.assertEqual(pformat(self.object), self.colorized_string)
+        self.assertIn(pformat(self.object), self.colorized_strings)
 
     @mock.patch('sys.platform', 'win32')
     @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
@@ -53,7 +63,7 @@ class TestDisplay(TestCase):
         isatty.return_value = True
         version.return_value = '10.0.14393'
         terminal_processing.return_value = True
-        self.assertEqual(pformat(self.object), self.colorized_string)
+        self.assertIn(pformat(self.object), self.colorized_strings)
 
     @mock.patch('sys.platform', 'linux')
     @mock.patch("sys.stdout.isatty")

tox.ini

@@ -32,7 +32,7 @@ download = true
 commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
 install_command =
-    pip install -U -ctests/upper-constraints.txt {opts} {packages}
+    python -I -m pip install -ctests/upper-constraints.txt {opts} {packages}
 
 [testenv:typing]
 basepython = python3
@@ -63,8 +63,7 @@ commands =
     flake8 {posargs:docs scrapy tests}
 
 [testenv:pylint]
-# reppy does not support Python 3.9+
-basepython = python3.8
+basepython = python3
 deps =
     {[testenv:extra-deps]deps}
     pylint==2.15.6
@@ -82,7 +81,7 @@ commands =
 
 [pinned]
 deps =
-    cryptography==3.3
+    cryptography==3.4.6
     cssselect==0.9.1
     h2==3.0
     itemadapter==0.1.0
@@ -108,7 +107,7 @@ deps =
 setenv =
     _SCRAPY_PINNED=true
 install_command =
-    pip install -U {opts} {packages}
+    python -I -m pip install {opts} {packages}
 
 [testenv:pinned]
 deps =
@@ -128,8 +127,7 @@ setenv =
     {[pinned]setenv}
 
 [testenv:extra-deps]
-# reppy does not support Python 3.9+
-basepython = python3.8
+basepython = python3
 deps =
     {[testenv]deps}
     boto
@@ -137,7 +135,6 @@ deps =
     # Twisted[http2] currently forces old mitmproxy because of h2 version
     # restrictions in their deps, so we need to pin old markupsafe here too.
     markupsafe < 2.1.0
-    reppy
     robotexclusionrulesparser
     Pillow>=4.0.0
     Twisted[http2]>=17.9.0