fix(shell): Run IPython prompt in a thread under a running loop (#7816)

pyproject.toml

@@ -69,8 +69,8 @@ brotli = [
 gcs = ["google-cloud-storage>=1.29.0"]
 httpx = ["httpx2[http2,socks]>=2.0.0"]
 images = ["Pillow>=8.3.2"]
-ipython = ["ipython>=7.1.0"]
-ptpython = ["ptpython>=2.0.1"]
+ipython = ["ipython>=8.15.0"]
+ptpython = ["ptpython>=3.0.23"]
 robotparser = ["robotexclusionrulesparser>=1.6.2"]
 s3 = ["boto3>=1.20.0"]
 twisted-http2 = ["Twisted[http2]>=21.7.0"]

scrapy/utils/console.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
+import asyncio
 import code
 from collections.abc import Callable
-from functools import wraps
+from functools import partial, wraps
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
@@ -16,16 +17,8 @@ def _embed_ipython_shell(
     namespace: dict[str, Any] | None = None, banner: str = ""
 ) -> EmbedFuncT:
     """Start an IPython Shell"""
-    try:
-        from IPython.terminal.embed import InteractiveShellEmbed  # noqa: T100,PLC0415
-        from IPython.terminal.ipapp import load_default_config  # noqa: PLC0415
-    except ImportError:
-        from IPython.frontend.terminal.embed import (  # type: ignore[import-not-found,no-redef]  # noqa: T100,PLC0415
-            InteractiveShellEmbed,
-        )
-        from IPython.frontend.terminal.ipapp import (  # type: ignore[import-not-found,no-redef]  # noqa: PLC0415
-            load_default_config,
-        )
+    from IPython.terminal.embed import InteractiveShellEmbed  # noqa: T100,PLC0415
+    from IPython.terminal.ipapp import load_default_config  # noqa: PLC0415
 
     @wraps(_embed_ipython_shell)
     def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
@@ -38,6 +31,19 @@ def _embed_ipython_shell(
         shell = InteractiveShellEmbed.instance(
             banner1=banner, user_ns=namespace, config=config
         )
+        # If an asyncio event loop is already running in this thread, e.g. when
+        # inspect_response() is called from a spider callback while using the
+        # asyncio reactor, prompt_toolkit cannot run its own event loop here, so
+        # ask it to run the prompt in a separate thread instead. pt_app is None
+        # when IPython falls back to its simple prompt, which needs no event loop.
+        # See https://github.com/scrapy/scrapy/issues/5447
+        if (pt_app := getattr(shell, "pt_app", None)) is not None:
+            try:
+                asyncio.get_running_loop()
+            except RuntimeError:
+                pass
+            else:
+                pt_app.prompt = partial(pt_app.prompt, in_thread=True)
         shell()
 
     return wrapper

tests/test_utils_console.py

@@ -1,10 +1,38 @@
 from __future__ import annotations
 
+import subprocess
+import sys
 from importlib.util import find_spec
+from io import BytesIO
+from typing import TYPE_CHECKING
 
 import pytest
+from pexpect import EOF
 
 from scrapy.utils.console import get_shell_embed_func, start_python_console
+from scrapy.utils.test import get_testenv
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+CONSOLE = """
+from scrapy.utils.console import start_python_console
+
+start_python_console(banner="SHELL-READY", shells=["ipython"])
+"""
+
+CONSOLE_IN_RUNNING_LOOP = """
+import asyncio
+
+from scrapy.utils.console import start_python_console
+
+
+async def main():
+    start_python_console(banner="SHELL-READY", shells=["ipython"])
+
+
+asyncio.run(main())
+"""
 
 
 def test_get_shell_embed_func():
@@ -61,6 +89,76 @@ def test_get_shell_embed_func_default():
     assert shell.__name__ == expected
 
 
+@pytest.mark.skipif(find_spec("IPython") is None, reason="IPython is not installed")
+class TestIPythonShell:
+    """Starting an IPython shell, with and without an asyncio event loop already
+    running in the calling thread. The latter happens when inspect_response() is
+    called from a spider callback while using the asyncio reactor."""
+
+    @staticmethod
+    def _env(tmp_path: Path) -> dict[str, str]:
+        env = get_testenv()
+        # Keep IPython away from the profile and history of the user running the tests.
+        env["IPYTHONDIR"] = str(tmp_path)
+        return env
+
+    def test_simple_prompt(self, tmp_path: Path) -> None:
+        """IPython falls back to its simple prompt, which needs no event loop,
+        when stdin is not a TTY."""
+        env = self._env(tmp_path)
+        p = subprocess.run(
+            [sys.executable, "-c", CONSOLE_IN_RUNNING_LOOP],
+            check=False,
+            capture_output=True,
+            encoding="utf-8",
+            timeout=60,
+            env=env,
+            stdin=subprocess.DEVNULL,
+        )
+        output = p.stdout + p.stderr
+        assert "SHELL-READY" in output
+        assert p.returncode == 0, output
+
+    @pytest.mark.skipif(
+        sys.platform == "win32", reason="requires a POSIX pseudo-terminal"
+    )
+    @pytest.mark.parametrize(
+        "script",
+        [CONSOLE, CONSOLE_IN_RUNNING_LOOP],
+        ids=["no_running_loop", "running_loop"],
+    )
+    def test_tty(self, tmp_path: Path, script: str) -> None:
+        """IPython uses prompt_toolkit, which needs an event loop of its own,
+        when stdin is a TTY."""
+        # pexpect only defines spawn, which needs a pseudo-terminal, on POSIX.
+        from pexpect import spawn  # noqa: PLC0415
+
+        env = self._env(tmp_path)
+        env.pop("IPY_TEST_SIMPLE_PROMPT", None)
+        env["TERM"] = "xterm"
+        logfile = BytesIO()
+        p = spawn(
+            sys.executable,
+            ["-c", script],
+            env=env,
+            timeout=60,
+        )
+        p.logfile_read = logfile
+        try:
+            # Wait for the prompt, which prompt_toolkit draws once it is done
+            # querying the terminal, before typing into it.
+            p.expect(r"In \[")
+            p.sendline("21*2")
+            p.expect_exact("42")
+            p.sendline("exit()")
+            p.expect(EOF)
+        finally:
+            p.close()
+        output = logfile.getvalue().decode()
+        assert "Traceback" not in output
+        assert p.exitstatus == 0, output
+
+
 def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None:
     def embed(namespace: dict[str, object], banner: str) -> None:
         raise SystemExit

tox.ini

@@ -190,8 +190,8 @@ deps =
     brotlicffi==1.2.0.0; implementation_name == "pypy"
     google-cloud-storage==1.29.0
     httpx2[http2,socks]==2.0.0
-    ipython==7.1.0
-    ptpython==2.0.1
+    ipython==8.15.0
+    ptpython==3.0.23
     robotexclusionrulesparser==1.6.2
     uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
     zstandard==0.16.0; implementation_name != "pypy"