From 94dff69468a9e734e10e0feea2674ae1ebf46bf7 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 11:15:58 +0200 Subject: [PATCH] Add VCS CI job, fix support for upcoming parsel version (#7924) --- .github/workflows/tests-vcs-deps.yml | 53 ++++++++++++++++++++++++++++ scrapy/selector/unified.py | 13 +++++-- tox.ini | 45 +++++++++++++++++++++++ 3 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/tests-vcs-deps.yml diff --git a/.github/workflows/tests-vcs-deps.yml b/.github/workflows/tests-vcs-deps.yml new file mode 100644 index 000000000..bf867dba7 --- /dev/null +++ b/.github/workflows/tests-vcs-deps.yml @@ -0,0 +1,53 @@ +name: VCS dependencies + +permissions: + contents: read + +on: + schedule: + - cron: '0 4 * * *' + workflow_dispatch: + +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + +jobs: + tests: + name: tests + runs-on: ubuntu-latest + env: + PYTEST_ADDOPTS: -n auto --no-cov + TOXENV: vcs-deps + UV_PYTHON_PREFERENCE: only-system + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + # Dependencies that ship wheels on PyPI are built from source here, so + # their build dependencies are needed: libxml2 and libxslt for lxml, + # libjpeg and zlib for Pillow, and autotools for the libuv bundled in + # uvloop. + - name: Install system libraries + run: | + sudo apt-get update + sudo apt-get install automake libjpeg-dev libtool libxml2-dev libxslt-dev zlib1g-dev + + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + run: uv tool install --python cpython mitmproxy + + - name: Run tests + run: uvx --with tox-uv tox diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index f6334c32c..fa91e2904 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -46,8 +46,10 @@ class Selector(_ParselSelector, object_ref): ``"json"``, ``"text"`` or ``None`` (default). It's passed to :class:`parsel.Selector` and its meaning is defined there. However, when ``type`` is ``None``, it is set to ``"xml"`` for an - :class:`~scrapy.http.XmlResponse` and to ``"html"`` otherwise before - passing it to :class:`parsel.Selector`. + :class:`~scrapy.http.XmlResponse` and to ``"html"`` for an + :class:`~scrapy.http.HtmlResponse` or for ``text`` before passing it to + :class:`parsel.Selector`, which for any other response is left to + determine the type from the response body. .. note:: JSON selector support requires ``parsel`` 1.8.0 or higher. With older versions setting ``type`` to ``"json"`` or ``"text"`` is not @@ -70,8 +72,13 @@ class Selector(_ParselSelector, object_ref): f"{self.__class__.__name__}.__init__() received both response and text" ) + # A response that is neither HTML nor XML, e.g. a JSON one, keeps type + # unset, so that parsel determines it from the body. if type is None: - type = "xml" if isinstance(response, XmlResponse) else "html" # noqa: A001 + if isinstance(response, XmlResponse): + type = "xml" # noqa: A001 + elif response is None or isinstance(response, HtmlResponse): + type = "html" # noqa: A001 if text is not None: response = _response_from_text(text, type) diff --git a/tox.ini b/tox.ini index c56ad011b..94c04f75f 100644 --- a/tox.ini +++ b/tox.ini @@ -201,6 +201,51 @@ setenv = {[min]setenv} commands = {[min]commands} +[testenv:vcs-deps] +basepython = python3 +deps = + {[testenv:extra-deps]deps} + uv +# Dependencies cap each other at their latest release, so their development +# branches usually cannot be resolved together: pyOpenSSL, for one, requires a +# cryptography older than the one cryptography itself is heading towards. +# --no-deps skips resolution entirely, replacing only these distributions and +# leaving the rest of the environment as the install above resolved it. +# +# Pillow and uvloop build from source, and need the libjpeg headers and +# autotools respectively. robotexclusionrulesparser has no public repository, +# so it stays at its latest release. +commands_pre = + uv pip install --python {envpython} --no-deps --reinstall \ + git+https://github.com/twisted/twisted \ + git+https://github.com/python-pillow/Pillow \ + git+https://github.com/MagicStack/uvloop \ + git+https://github.com/pyca/cryptography \ + git+https://github.com/scrapy/cssselect \ + git+https://github.com/tiran/defusedxml \ + git+https://github.com/scrapy/itemadapter \ + git+https://github.com/scrapy/itemloaders \ + git+https://github.com/lxml/lxml \ + git+https://github.com/pypa/packaging \ + git+https://github.com/scrapy/parsel \ + git+https://github.com/scrapy/protego \ + git+https://github.com/pyca/pyopenssl \ + git+https://github.com/scrapy/queuelib \ + git+https://github.com/pyca/service-identity \ + git+https://github.com/john-kurkowski/tldextract \ + git+https://github.com/scrapy/w3lib \ + git+https://github.com/zopefoundation/zope.interface \ + git+https://github.com/mcfletch/pydispatcher \ + git+https://github.com/boto/boto3 \ + git+https://github.com/bpython/bpython \ + git+https://github.com/google/brotli \ + git+https://github.com/python-hyper/brotlicffi \ + git+https://github.com/googleapis/python-storage \ + git+https://github.com/pydantic/httpx2\#subdirectory=src/httpx2 \ + git+https://github.com/ipython/ipython \ + git+https://github.com/prompt-toolkit/ptpython \ + git+https://github.com/indygreg/python-zstandard + [testenv:default-reactor] commands = {[testenv]commands} --reactor=default