Use external mitmproxy.

.github/workflows/tests-ubuntu.yml

@@ -79,9 +79,6 @@ jobs:
         - python-version: "3.14"
           env:
             TOXENV: botocore
-        - python-version: "3.14"
-          env:
-            TOXENV: mitmproxy
 
     steps:
     - uses: actions/checkout@v6
@@ -97,6 +94,9 @@ jobs:
         sudo apt-get update
         sudo apt-get install libxml2-dev libxslt-dev
 
+    - name: Install mitmproxy
+      run: pipx install mitmproxy
+
     - name: Run tests
       env: ${{ matrix.env }}
       run: |

conftest.py

@@ -11,7 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policy
 from scrapy.utils.reactorless import install_reactor_import_hook
 from tests.keys import generate_keys
 from tests.mockserver.http import MockServer
-from tests.mockserver.mitm_proxy import MitmProxy
+from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd
 
 if TYPE_CHECKING:
     from collections.abc import Generator
@@ -127,7 +127,6 @@ def pytest_runtest_setup(item):
         "uvloop",
         "botocore",
         "boto3",
-        "mitmproxy",
     ]
 
     for module in optional_deps:
@@ -137,6 +136,9 @@ def pytest_runtest_setup(item):
             except ImportError:
                 pytest.skip(f"{module} is not installed")
 
+    if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None:
+        pytest.skip("mitmdump is not available")
+
 
 # Generate localhost certificate files, needed by some tests
 generate_keys()

docs/contributing.rst

@@ -371,6 +371,21 @@ To see coverage report install :doc:`coverage <coverage:index>`
 
 see output of ``coverage --help`` for more options like html or xml report.
 
+Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a
+fully featured proxy server; they are skipped when one cannot be found
+(``mitmproxy`` is intentionally not a test dependency that would be installed
+into test venvs, as that sometimes leads to various dependency conflicts).
+To run these tests, make ``mitmdump`` available in one of these ways:
+
+* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with
+  pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``);
+
+* have uv_ installed, in which case the tests will run
+  ``uvx --from mitmproxy mitmdump``;
+
+* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump``
+  executable.
+
 Writing tests
 -------------
 
@@ -398,3 +413,6 @@ And their unit-tests are in::
 .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
 .. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
+.. _mitmproxy: https://mitmproxy.org/
+.. _pipx: https://pipx.pypa.io/
+.. _uv: https://docs.astral.sh/uv/

pyproject.toml

@@ -269,7 +269,7 @@ markers = [
     "requires_uvloop: marks tests as only enabled when uvloop is known to be working",
     "requires_botocore: marks tests that need botocore (but not boto3)",
     "requires_boto3: marks tests that need botocore and boto3",
-    "requires_mitmproxy: marks tests that need mitmproxy",
+    "requires_mitmproxy: marks tests that need a mitmdump executable",
     "requires_internet: marks tests that need real Internet access",
 ]
 filterwarnings = [

tests/mockserver/mitm_proxy.py

@@ -1,12 +1,38 @@
 from __future__ import annotations
 
+import functools
+import os
 import re
-import sys
+import shutil
 from pathlib import Path
 from subprocess import PIPE, Popen
 from urllib.parse import urlsplit, urlunsplit
 
 
+@functools.cache
+def mitmdump_cmd() -> list[str] | None:
+    """Return the command prefix used to invoke ``mitmdump``, or ``None`` if it
+    cannot be resolved.
+
+    We don't want to install ``mitmproxy`` into the test env (it has a lot of
+    dependencies that can conflict with some of the Scrapy/test ones, and its
+    newer versions may not support older Python versions). So we expect it
+    installed externally. We look for the ``mitmdump`` binary in the following
+    sources:
+
+    1. the ``MITMDUMP`` environment variable;
+    2. a ``mitmdump`` binary on ``PATH``;
+    3. using ``uvx --from mitmproxy mitmdump`` if ``uvx`` is available.
+    """
+    if env := os.environ.get("MITMDUMP"):
+        return [env]
+    if path := shutil.which("mitmdump"):
+        return [path]
+    if uvx := shutil.which("uvx"):
+        return [uvx, "--from", "mitmproxy", "mitmdump"]
+    return None
+
+
 class MitmProxy:
     auth_user = "scrapy"
     auth_pass = "scrapy"
@@ -15,12 +41,11 @@ class MitmProxy:
         self.mode = mode
 
     def start(self) -> str:
-        script = """
-import sys
-from mitmproxy.tools.main import mitmdump
-sys.argv[0] = "mitmdump"
-sys.exit(mitmdump())
-        """
+        cmd = mitmdump_cmd()
+        if not cmd:
+            raise RuntimeError(
+                "mitmdump is not available. Please install mitmproxy or uv."
+            )
         cert_path = Path(__file__).parent.parent.resolve() / "keys"
         args = [
             "--listen-host",
@@ -38,15 +63,10 @@ sys.exit(mitmdump())
         if self.mode:
             args += ["--mode", self.mode]
         self.proc: Popen[str] = Popen(
-            [
-                sys.executable,
-                "-u",
-                "-c",
-                script,
-                *args,
-            ],
+            [*cmd, *args],
             stdout=PIPE,
             text=True,
+            env={**os.environ, "PYTHONUNBUFFERED": "1"},
         )
         assert self.proc.stdout is not None
         scheme = "socks5" if self.mode == "socks5" else "http"

tox.ini

@@ -28,7 +28,6 @@ envlist =
     no-reactor
     no-reactor-extra-deps
     botocore
-    mitmproxy
     pypy3
     pypy3-extra-deps
 minversion = 1.7.0
@@ -55,6 +54,7 @@ deps =
 passenv =
     PYTHONTRACEMALLOC
     PYTEST_ADDOPTS
+    MITMDUMP
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY
@@ -319,17 +319,3 @@ setenv =
     {[min]setenv}
 commands =
     pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore
-
-
-# Run proxy tests that use mitmproxy in a separate env to avoid installing
-# numerous mitmproxy deps in other envs (even in extra-deps), as they can
-# conflict with other deps we want, or don't want, to have installed there.
-
-[testenv:mitmproxy]
-deps =
-    {[testenv]deps}
-    # mitmproxy does not support PyPy
-    mitmproxy; implementation_name != "pypy"
-    httpx[http2,socks]
-commands =
-    pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=mitmproxy.junit.xml -o junit_family=legacy} -m requires_mitmproxy