Merge remote-tracking branch 'origin/master' into improve-coverage-utils

# Conflicts:
#	pyproject.toml
#	tox.ini

docs/topics/asyncio.rst

@@ -267,6 +267,7 @@ Here are some examples of APIs and patterns that need a replacement:
 
 Scrapy provides unified helpers for some of these examples:
 
+.. autofunction:: scrapy.utils.asyncio.sleep
 .. autofunction:: scrapy.utils.asyncio.call_later
 .. autofunction:: scrapy.utils.asyncio.create_looping_call
 .. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall

docs/topics/settings.rst

@@ -575,7 +575,7 @@ CONCURRENT_REQUESTS
 Default: ``16``
 
 The maximum number of concurrent (i.e. simultaneous) requests that will be
-performed by the Scrapy downloader.
+performed by the Scrapy downloader. Use ``0`` for no limit.
 
 .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
 

pyproject.toml

@@ -69,8 +69,8 @@ brotli = [
 gcs = ["google-cloud-storage>=1.29.0"]
 httpx = ["httpx2[http2,socks]>=2.0.0"]
 images = ["Pillow>=8.3.2"]
-ipython = ["ipython>=7.3.0"]
-ptpython = ["ptpython>=2.0.1"]
+ipython = ["ipython>=8.15.0"]
+ptpython = ["ptpython>=3.0.23"]
 robotparser = ["robotexclusionrulesparser>=1.6.2"]
 s3 = ["boto3>=1.20.0"]
 twisted-http2 = ["Twisted[http2]>=21.7.0"]

scrapy/core/downloader/__init__.py

@@ -138,7 +138,8 @@ class Downloader:
             self.active.remove(request)
 
     def needs_backout(self) -> bool:
-        return len(self.active) >= self.total_concurrency
+        # A total concurrency of 0 means no limit.
+        return 0 < self.total_concurrency <= len(self.active)
 
     @_warn_spider_arg
     def _get_slot(

scrapy/core/downloader/handlers/_httpx.py

@@ -92,10 +92,11 @@ class HttpxDownloadHandler(_Base):
         self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
         self._bind_host: str | None = self._get_bind_address_host()
         self._limits: httpx.Limits = httpx.Limits(
-            # hard limit on simultaneous connections
-            max_connections=self._pool_size_total,
+            # hard limit on simultaneous connections (None for no limit, which
+            # is what a CONCURRENT_REQUESTS of 0 means)
+            max_connections=self._pool_size_total or None,
             # total number of idle connections in the pool (extra ones are closed)
-            max_keepalive_connections=self._pool_size_total,
+            max_keepalive_connections=self._pool_size_total or None,
         )
 
         self._default_client: httpx.AsyncClient = self._make_client()

scrapy/utils/asyncio.py

@@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
 from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar
 
 from twisted.internet.defer import Deferred
-from twisted.internet.task import LoopingCall
+from twisted.internet.task import LoopingCall, deferLater
 from twisted.internet.threads import deferToThread
 
 from scrapy.utils.asyncgen import as_async_generator
@@ -293,6 +293,24 @@ class CallLaterResult:
             self._delayed_call = None
 
 
+async def sleep(seconds: float) -> None:
+    """Sleep for *seconds*.
+
+    .. versionadded:: VERSION
+
+    This uses either :func:`asyncio.sleep` or
+    :func:`~twisted.internet.task.deferLater`, depending on whether asyncio
+    support is available.
+    """
+    if is_asyncio_available():
+        await asyncio.sleep(seconds)
+        return
+
+    from twisted.internet import reactor
+
+    await deferLater(reactor, seconds)
+
+
 async def run_in_thread(
     func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
 ) -> _T:

scrapy/utils/console.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
+import asyncio
 import code
 from collections.abc import Callable
-from functools import wraps
+from functools import partial, wraps
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
@@ -16,16 +17,8 @@ def _embed_ipython_shell(
     namespace: dict[str, Any] | None = None, banner: str = ""
 ) -> EmbedFuncT:
     """Start an IPython Shell"""
-    try:
-        from IPython.terminal.embed import InteractiveShellEmbed  # noqa: T100,PLC0415
-        from IPython.terminal.ipapp import load_default_config  # noqa: PLC0415
-    except ImportError:
-        from IPython.frontend.terminal.embed import (  # type: ignore[import-not-found,no-redef]  # noqa: T100,PLC0415
-            InteractiveShellEmbed,
-        )
-        from IPython.frontend.terminal.ipapp import (  # type: ignore[import-not-found,no-redef]  # noqa: PLC0415
-            load_default_config,
-        )
+    from IPython.terminal.embed import InteractiveShellEmbed  # noqa: T100,PLC0415
+    from IPython.terminal.ipapp import load_default_config  # noqa: PLC0415
 
     @wraps(_embed_ipython_shell)
     def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
@@ -38,6 +31,19 @@ def _embed_ipython_shell(
         shell = InteractiveShellEmbed.instance(
             banner1=banner, user_ns=namespace, config=config
         )
+        # If an asyncio event loop is already running in this thread, e.g. when
+        # inspect_response() is called from a spider callback while using the
+        # asyncio reactor, prompt_toolkit cannot run its own event loop here, so
+        # ask it to run the prompt in a separate thread instead. pt_app is None
+        # when IPython falls back to its simple prompt, which needs no event loop.
+        # See https://github.com/scrapy/scrapy/issues/5447
+        if (pt_app := getattr(shell, "pt_app", None)) is not None:
+            try:
+                asyncio.get_running_loop()
+            except RuntimeError:
+                pass
+            else:
+                pt_app.prompt = partial(pt_app.prompt, in_thread=True)
         shell()
 
     return wrapper

scrapy/utils/defer.py

@@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator
 from twisted.python import failure
 
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncio import is_asyncio_available
+from scrapy.utils.asyncio import is_asyncio_available, sleep
 from scrapy.utils.python import global_object_name
 
 if TYPE_CHECKING:
@@ -90,14 +90,7 @@ async def _defer_sleep_async() -> None:
     """Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers
     before attending pending delayed calls, so do not set delay to zero.
     """
-    if is_asyncio_available():
-        await asyncio.sleep(_DEFER_DELAY)
-    else:
-        from twisted.internet import reactor
-
-        d: Deferred[None] = Deferred()
-        reactor.callLater(_DEFER_DELAY, d.callback, None)
-        await d
+    await sleep(_DEFER_DELAY)
 
 
 def defer_result(result: Any) -> Deferred[Any]:  # pragma: no cover

tests/test_core_downloader.py

@@ -14,6 +14,7 @@ from twisted.web import server, static
 from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
 from twisted.web.client import Response as TxResponse
 
+from scrapy import Request
 from scrapy.core.downloader import Downloader, Slot, tls
 from scrapy.core.downloader.contextfactory import (
     _load_context_factory_from_settings,
@@ -296,6 +297,23 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
         await self._assert_factory_works(server_url, client_context_factory)
 
 
+@pytest.mark.parametrize(
+    ("concurrency", "active", "expected"),
+    [
+        (2, 1, False),
+        (2, 2, True),
+        (0, 0, False),
+        (0, 2, False),
+    ],
+)
+def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:
+    crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
+    downloader = Downloader(crawler)
+    downloader.active = {Request(f"https://example.com/{i}") for i in range(active)}
+    assert downloader.needs_backout() is expected
+    downloader.close()
+
+
 @coroutine_test
 async def test_fetch_deprecated_spider_arg():
     class CustomDownloader(Downloader):

tests/test_crawler_subprocess.py

@@ -14,7 +14,8 @@ from packaging.version import parse as parse_version
 from pexpect.popen_spawn import PopenSpawn
 from w3lib import __version__ as w3lib_version
 
-from tests.utils import async_sleep, get_script_run_env
+from scrapy.utils.asyncio import sleep
+from tests.utils import get_script_run_env
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
@@ -248,7 +249,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         p.kill(sig)
         p.expect_exact("shutting down gracefully")
         # sending the second signal too fast often causes problems
-        await async_sleep(0.01)
+        await sleep(0.01)
         p.kill(sig)
         p.expect_exact("forcing unclean shutdown")
         p.wait()  # type: ignore[no-untyped-call]

tests/test_downloader_handler_httpx.py

@@ -15,6 +15,8 @@ from scrapy.core.downloader.handlers._httpx import (
     HttpxDownloadHandler,
 )
 from scrapy.exceptions import DownloadFailedError
+from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.test import get_crawler
 from tests.utils.bases.download_handlers_http import (
     TestHttpBase,
     TestHttpProxyBase,
@@ -166,3 +168,15 @@ class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
 @pytest.mark.requires_internet
 class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase):
     pass
+
+
+@pytest.mark.parametrize(("concurrency", "expected"), [(16, 16), (0, None)])
+@coroutine_test
+async def test_pool_limits(concurrency: int, expected: int | None) -> None:
+    crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
+    handler = build_from_crawler(HttpxDownloadHandler, crawler)
+    try:
+        assert handler._limits.max_connections == expected
+        assert handler._limits.max_keepalive_connections == expected
+    finally:
+        await handler.close()

tests/test_engine_loop.py

@@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Any
 
 from scrapy import Request, Spider, signals
 from scrapy.core.scheduler import BaseScheduler
-from scrapy.utils.asyncio import call_later
+from scrapy.utils.asyncio import call_later, sleep
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
-from tests.utils import async_sleep
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
@@ -65,23 +64,23 @@ class TestMain:
             async def start(self):
                 yield Request("data:,a")
 
-                await async_sleep(seconds)
+                await sleep(seconds)
 
                 self.crawler.engine._slot.scheduler.pause()
                 self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))
 
                 # During this time, the scheduler reports having requests but
                 # returns None.
-                await async_sleep(seconds)
+                await sleep(seconds)
 
                 self.crawler.engine._slot.scheduler.unpause()
 
                 # The scheduler request is processed.
-                await async_sleep(seconds)
+                await sleep(seconds)
 
                 yield Request("data:,c")
 
-                await async_sleep(seconds)
+                await sleep(seconds)
 
                 self.crawler.engine._slot.scheduler.pause()
                 self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d"))

tests/test_utils_asyncio.py

@@ -12,7 +12,9 @@ from scrapy.utils.asyncgen import as_async_generator
 from scrapy.utils.asyncio import (
     AsyncioLoopingCall,
     _parallel_asyncio,
+    call_later,
     is_asyncio_available,
+    sleep,
 )
 from tests.utils.decorators import coroutine_test
 
@@ -26,6 +28,15 @@ async def test_is_asyncio_available(reactor_pytest: str) -> None:
     assert is_asyncio_available() == (reactor_pytest != "default")
 
 
+@coroutine_test
+async def test_sleep() -> None:
+    events: list[str] = []
+    call_later(0.05, events.append, "call_later")
+    await sleep(0.1)
+    events.append("sleep")
+    assert events == ["call_later", "sleep"]
+
+
 @pytest.mark.only_asyncio
 class TestParallelAsyncio:
     """Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync."""

tests/test_utils_console.py

@@ -1,10 +1,38 @@
 from __future__ import annotations
 
+import subprocess
+import sys
 from importlib.util import find_spec
+from io import BytesIO
+from typing import TYPE_CHECKING
 
 import pytest
+from pexpect import EOF
 
 from scrapy.utils.console import get_shell_embed_func, start_python_console
+from scrapy.utils.test import get_testenv
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+CONSOLE = """
+from scrapy.utils.console import start_python_console
+
+start_python_console(banner="SHELL-READY", shells=["ipython"])
+"""
+
+CONSOLE_IN_RUNNING_LOOP = """
+import asyncio
+
+from scrapy.utils.console import start_python_console
+
+
+async def main():
+    start_python_console(banner="SHELL-READY", shells=["ipython"])
+
+
+asyncio.run(main())
+"""
 
 
 def test_get_shell_embed_func():
@@ -70,6 +98,76 @@ def test_get_shell_embed_func_default():
     assert shell.__name__ == expected
 
 
+@pytest.mark.skipif(find_spec("IPython") is None, reason="IPython is not installed")
+class TestIPythonShell:
+    """Starting an IPython shell, with and without an asyncio event loop already
+    running in the calling thread. The latter happens when inspect_response() is
+    called from a spider callback while using the asyncio reactor."""
+
+    @staticmethod
+    def _env(tmp_path: Path) -> dict[str, str]:
+        env = get_testenv()
+        # Keep IPython away from the profile and history of the user running the tests.
+        env["IPYTHONDIR"] = str(tmp_path)
+        return env
+
+    def test_simple_prompt(self, tmp_path: Path) -> None:
+        """IPython falls back to its simple prompt, which needs no event loop,
+        when stdin is not a TTY."""
+        env = self._env(tmp_path)
+        p = subprocess.run(
+            [sys.executable, "-c", CONSOLE_IN_RUNNING_LOOP],
+            check=False,
+            capture_output=True,
+            encoding="utf-8",
+            timeout=60,
+            env=env,
+            stdin=subprocess.DEVNULL,
+        )
+        output = p.stdout + p.stderr
+        assert "SHELL-READY" in output
+        assert p.returncode == 0, output
+
+    @pytest.mark.skipif(
+        sys.platform == "win32", reason="requires a POSIX pseudo-terminal"
+    )
+    @pytest.mark.parametrize(
+        "script",
+        [CONSOLE, CONSOLE_IN_RUNNING_LOOP],
+        ids=["no_running_loop", "running_loop"],
+    )
+    def test_tty(self, tmp_path: Path, script: str) -> None:
+        """IPython uses prompt_toolkit, which needs an event loop of its own,
+        when stdin is a TTY."""
+        # pexpect only defines spawn, which needs a pseudo-terminal, on POSIX.
+        from pexpect import spawn  # noqa: PLC0415
+
+        env = self._env(tmp_path)
+        env.pop("IPY_TEST_SIMPLE_PROMPT", None)
+        env["TERM"] = "xterm"
+        logfile = BytesIO()
+        p = spawn(
+            sys.executable,
+            ["-c", script],
+            env=env,
+            timeout=60,
+        )
+        p.logfile_read = logfile
+        try:
+            # Wait for the prompt, which prompt_toolkit draws once it is done
+            # querying the terminal, before typing into it.
+            p.expect(r"In \[")
+            p.sendline("21*2")
+            p.expect_exact("42")
+            p.sendline("exit()")
+            p.expect(EOF)
+        finally:
+            p.close()
+        output = logfile.getvalue().decode()
+        assert "Traceback" not in output
+        assert p.exitstatus == 0, output
+
+
 def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None:
     def embed(namespace: dict[str, object], banner: str) -> None:
         raise SystemExit

tests/utils/__init__.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import asyncio
 import os
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -8,8 +7,6 @@ from typing import TYPE_CHECKING
 from twisted.internet.defer import Deferred
 
 from scrapy.settings import Settings, default_settings
-from scrapy.utils.asyncio import is_asyncio_available
-from scrapy.utils.defer import maybe_deferred_to_future
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -23,13 +20,6 @@ def twisted_sleep(seconds: float):
     return d
 
 
-async def async_sleep(seconds: float) -> None:
-    if is_asyncio_available():
-        await asyncio.sleep(seconds)
-    else:
-        await maybe_deferred_to_future(twisted_sleep(seconds))
-
-
 def get_script_run_env() -> dict[str, str]:
     """Return a OS environment dict suitable to run scripts shipped with tests."""
 

tox.ini

@@ -190,8 +190,8 @@ deps =
     brotlicffi==1.2.0.0; implementation_name == "pypy"
     google-cloud-storage==1.29.0
     httpx2[http2,socks]==2.0.0
-    ipython==7.3.0
-    ptpython==2.0.1
+    ipython==8.15.0
+    ptpython==3.0.23
     robotexclusionrulesparser==1.6.2
     uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
     zstandard==0.16.0; implementation_name != "pypy"