Add VCS CI job, fix support for upcoming parsel version (#7924)

.github/workflows/tests-vcs-deps.yml

@@ -0,0 +1,53 @@
+name: VCS dependencies
+
+permissions:
+  contents: read
+
+on:
+  schedule:
+  - cron: '0 4 * * *'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{github.workflow}}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    name: tests
+    runs-on: ubuntu-latest
+    env:
+      PYTEST_ADDOPTS: -n auto --no-cov
+      TOXENV: vcs-deps
+      UV_PYTHON_PREFERENCE: only-system
+    steps:
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      with:
+        persist-credentials: false
+
+    - name: Set up Python
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+      with:
+        python-version: "3.14"
+
+    # Dependencies that ship wheels on PyPI are built from source here, so
+    # their build dependencies are needed: libxml2 and libxslt for lxml,
+    # libjpeg and zlib for Pillow, and autotools for the libuv bundled in
+    # uvloop.
+    - name: Install system libraries
+      run: |
+        sudo apt-get update
+        sudo apt-get install automake libjpeg-dev libtool libxml2-dev libxslt-dev zlib1g-dev
+
+    - name: Set up uv
+      uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+      with:
+        cache-dependency-glob: |
+          pyproject.toml
+          tox.ini
+
+    - name: Install mitmproxy
+      run: uv tool install --python cpython mitmproxy
+
+    - name: Run tests
+      run: uvx --with tox-uv tox

scrapy/selector/unified.py

@@ -46,8 +46,10 @@ class Selector(_ParselSelector, object_ref):
     ``"json"``, ``"text"`` or ``None`` (default). It's passed to
     :class:`parsel.Selector` and its meaning is defined there. However, when
     ``type`` is ``None``, it is set to ``"xml"`` for an
-    :class:`~scrapy.http.XmlResponse` and to ``"html"`` otherwise before
-    passing it to :class:`parsel.Selector`.
+    :class:`~scrapy.http.XmlResponse` and to ``"html"`` for an
+    :class:`~scrapy.http.HtmlResponse` or for ``text`` before passing it to
+    :class:`parsel.Selector`, which for any other response is left to
+    determine the type from the response body.
 
     .. note:: JSON selector support requires ``parsel`` 1.8.0 or higher. With
        older versions setting ``type`` to ``"json"`` or ``"text"`` is not
@@ -70,8 +72,13 @@ class Selector(_ParselSelector, object_ref):
                 f"{self.__class__.__name__}.__init__() received both response and text"
             )
 
+        # A response that is neither HTML nor XML, e.g. a JSON one, keeps type
+        # unset, so that parsel determines it from the body.
         if type is None:
-            type = "xml" if isinstance(response, XmlResponse) else "html"  # noqa: A001
+            if isinstance(response, XmlResponse):
+                type = "xml"  # noqa: A001
+            elif response is None or isinstance(response, HtmlResponse):
+                type = "html"  # noqa: A001
 
         if text is not None:
             response = _response_from_text(text, type)

tox.ini

@@ -201,6 +201,51 @@ setenv =
     {[min]setenv}
 commands = {[min]commands}
 
+[testenv:vcs-deps]
+basepython = python3
+deps =
+    {[testenv:extra-deps]deps}
+    uv
+# Dependencies cap each other at their latest release, so their development
+# branches usually cannot be resolved together: pyOpenSSL, for one, requires a
+# cryptography older than the one cryptography itself is heading towards.
+# --no-deps skips resolution entirely, replacing only these distributions and
+# leaving the rest of the environment as the install above resolved it.
+#
+# Pillow and uvloop build from source, and need the libjpeg headers and
+# autotools respectively. robotexclusionrulesparser has no public repository,
+# so it stays at its latest release.
+commands_pre =
+    uv pip install --python {envpython} --no-deps --reinstall \
+        git+https://github.com/twisted/twisted \
+        git+https://github.com/python-pillow/Pillow \
+        git+https://github.com/MagicStack/uvloop \
+        git+https://github.com/pyca/cryptography \
+        git+https://github.com/scrapy/cssselect \
+        git+https://github.com/tiran/defusedxml \
+        git+https://github.com/scrapy/itemadapter \
+        git+https://github.com/scrapy/itemloaders \
+        git+https://github.com/lxml/lxml \
+        git+https://github.com/pypa/packaging \
+        git+https://github.com/scrapy/parsel \
+        git+https://github.com/scrapy/protego \
+        git+https://github.com/pyca/pyopenssl \
+        git+https://github.com/scrapy/queuelib \
+        git+https://github.com/pyca/service-identity \
+        git+https://github.com/john-kurkowski/tldextract \
+        git+https://github.com/scrapy/w3lib \
+        git+https://github.com/zopefoundation/zope.interface \
+        git+https://github.com/mcfletch/pydispatcher \
+        git+https://github.com/boto/boto3 \
+        git+https://github.com/bpython/bpython \
+        git+https://github.com/google/brotli \
+        git+https://github.com/python-hyper/brotlicffi \
+        git+https://github.com/googleapis/python-storage \
+        git+https://github.com/pydantic/httpx2\#subdirectory=src/httpx2 \
+        git+https://github.com/ipython/ipython \
+        git+https://github.com/prompt-toolkit/ptpython \
+        git+https://github.com/indygreg/python-zstandard
+
 [testenv:default-reactor]
 commands =
     {[testenv]commands} --reactor=default