From d25a95d2f458183546e14851f58c2c17a097ee0c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 5 Jul 2026 23:12:10 +0500 Subject: [PATCH] Use external mitmproxy. --- .github/workflows/tests-ubuntu.yml | 6 ++-- conftest.py | 6 ++-- docs/contributing.rst | 18 +++++++++++ pyproject.toml | 2 +- tests/mockserver/mitm_proxy.py | 48 +++++++++++++++++++++--------- tox.ini | 16 +--------- 6 files changed, 61 insertions(+), 35 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f51dd9799..15f25d9b8 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -79,9 +79,6 @@ jobs: - python-version: "3.14" env: TOXENV: botocore - - python-version: "3.14" - env: - TOXENV: mitmproxy steps: - uses: actions/checkout@v6 @@ -97,6 +94,9 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev + - name: Install mitmproxy + run: pipx install mitmproxy + - name: Run tests env: ${{ matrix.env }} run: | diff --git a/conftest.py b/conftest.py index 532f83f56..7403b07b2 100644 --- a/conftest.py +++ b/conftest.py @@ -11,7 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policy from scrapy.utils.reactorless import install_reactor_import_hook from tests.keys import generate_keys from tests.mockserver.http import MockServer -from tests.mockserver.mitm_proxy import MitmProxy +from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd if TYPE_CHECKING: from collections.abc import Generator @@ -127,7 +127,6 @@ def pytest_runtest_setup(item): "uvloop", "botocore", "boto3", - "mitmproxy", ] for module in optional_deps: @@ -137,6 +136,9 @@ def pytest_runtest_setup(item): except ImportError: pytest.skip(f"{module} is not installed") + if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None: + pytest.skip("mitmdump is not available") + # Generate localhost certificate files, needed by some tests generate_keys() diff --git a/docs/contributing.rst b/docs/contributing.rst index c868a0ac4..32dd989a5 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -371,6 +371,21 @@ To see coverage report install :doc:`coverage ` see output of ``coverage --help`` for more options like html or xml report. +Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a +fully featured proxy server; they are skipped when one cannot be found +(``mitmproxy`` is intentionally not a test dependency that would be installed +into test venvs, as that sometimes leads to various dependency conflicts). +To run these tests, make ``mitmdump`` available in one of these ways: + +* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with + pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``); + +* have uv_ installed, in which case the tests will run + ``uvx --from mitmproxy mitmdump``; + +* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump`` + executable. + Writing tests ------------- @@ -398,3 +413,6 @@ And their unit-tests are in:: .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 .. _test coverage: https://app.codecov.io/gh/scrapy/scrapy +.. _mitmproxy: https://mitmproxy.org/ +.. _pipx: https://pipx.pypa.io/ +.. _uv: https://docs.astral.sh/uv/ diff --git a/pyproject.toml b/pyproject.toml index b8d9067f9..9644e73c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -269,7 +269,7 @@ markers = [ "requires_uvloop: marks tests as only enabled when uvloop is known to be working", "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", - "requires_mitmproxy: marks tests that need mitmproxy", + "requires_mitmproxy: marks tests that need a mitmdump executable", "requires_internet: marks tests that need real Internet access", ] filterwarnings = [ diff --git a/tests/mockserver/mitm_proxy.py b/tests/mockserver/mitm_proxy.py index 56620f84e..f1c2f930e 100644 --- a/tests/mockserver/mitm_proxy.py +++ b/tests/mockserver/mitm_proxy.py @@ -1,12 +1,38 @@ from __future__ import annotations +import functools +import os import re -import sys +import shutil from pathlib import Path from subprocess import PIPE, Popen from urllib.parse import urlsplit, urlunsplit +@functools.cache +def mitmdump_cmd() -> list[str] | None: + """Return the command prefix used to invoke ``mitmdump``, or ``None`` if it + cannot be resolved. + + We don't want to install ``mitmproxy`` into the test env (it has a lot of + dependencies that can conflict with some of the Scrapy/test ones, and its + newer versions may not support older Python versions). So we expect it + installed externally. We look for the ``mitmdump`` binary in the following + sources: + + 1. the ``MITMDUMP`` environment variable; + 2. a ``mitmdump`` binary on ``PATH``; + 3. using ``uvx --from mitmproxy mitmdump`` if ``uvx`` is available. + """ + if env := os.environ.get("MITMDUMP"): + return [env] + if path := shutil.which("mitmdump"): + return [path] + if uvx := shutil.which("uvx"): + return [uvx, "--from", "mitmproxy", "mitmdump"] + return None + + class MitmProxy: auth_user = "scrapy" auth_pass = "scrapy" @@ -15,12 +41,11 @@ class MitmProxy: self.mode = mode def start(self) -> str: - script = """ -import sys -from mitmproxy.tools.main import mitmdump -sys.argv[0] = "mitmdump" -sys.exit(mitmdump()) - """ + cmd = mitmdump_cmd() + if not cmd: + raise RuntimeError( + "mitmdump is not available. Please install mitmproxy or uv." + ) cert_path = Path(__file__).parent.parent.resolve() / "keys" args = [ "--listen-host", @@ -38,15 +63,10 @@ sys.exit(mitmdump()) if self.mode: args += ["--mode", self.mode] self.proc: Popen[str] = Popen( - [ - sys.executable, - "-u", - "-c", - script, - *args, - ], + [*cmd, *args], stdout=PIPE, text=True, + env={**os.environ, "PYTHONUNBUFFERED": "1"}, ) assert self.proc.stdout is not None scheme = "socks5" if self.mode == "socks5" else "http" diff --git a/tox.ini b/tox.ini index 5017d7636..d29d2f040 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,6 @@ envlist = no-reactor no-reactor-extra-deps botocore - mitmproxy pypy3 pypy3-extra-deps minversion = 1.7.0 @@ -55,6 +54,7 @@ deps = passenv = PYTHONTRACEMALLOC PYTEST_ADDOPTS + MITMDUMP S3_TEST_FILE_URI AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY @@ -319,17 +319,3 @@ setenv = {[min]setenv} commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore - - -# Run proxy tests that use mitmproxy in a separate env to avoid installing -# numerous mitmproxy deps in other envs (even in extra-deps), as they can -# conflict with other deps we want, or don't want, to have installed there. - -[testenv:mitmproxy] -deps = - {[testenv]deps} - # mitmproxy does not support PyPy - mitmproxy; implementation_name != "pypy" - httpx[http2,socks] -commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=mitmproxy.junit.xml -o junit_family=legacy} -m requires_mitmproxy