diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index fa91e2904..0bd6d49a2 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -4,7 +4,7 @@ from typing import Any, Literal from parsel import Selector as _ParselSelector -from scrapy.http import HtmlResponse, TextResponse, XmlResponse +from scrapy.http import HtmlResponse, JsonResponse, TextResponse, XmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import get_base_url from scrapy.utils.trackref import object_ref @@ -34,6 +34,11 @@ class Selector(_ParselSelector, object_ref): An instance of :class:`Selector` is a wrapper over response to select certain parts of its content. + .. versionchanged:: VERSION + The type of a :class:`~scrapy.http.JsonResponse` selector is now + ``json``, and the type of the selector of any other response that is + neither HTML nor XML is determined from the response body. + ``response`` is an :class:`~scrapy.http.HtmlResponse` or an :class:`~scrapy.http.XmlResponse` object that will be used for selecting and extracting data. @@ -46,11 +51,20 @@ class Selector(_ParselSelector, object_ref): ``"json"``, ``"text"`` or ``None`` (default). It's passed to :class:`parsel.Selector` and its meaning is defined there. However, when ``type`` is ``None``, it is set to ``"xml"`` for an - :class:`~scrapy.http.XmlResponse` and to ``"html"`` for an + :class:`~scrapy.http.XmlResponse`, to ``"json"`` for a + :class:`~scrapy.http.JsonResponse` and to ``"html"`` for an :class:`~scrapy.http.HtmlResponse` or for ``text`` before passing it to :class:`parsel.Selector`, which for any other response is left to determine the type from the response body. + The response class, and hence the selector type, comes from the content + type that the website reports. When a website reports the wrong content + type, recast the response into the right class: + + .. code-block:: python + + response = response.replace(cls=HtmlResponse) + .. note:: JSON selector support requires ``parsel`` 1.8.0 or higher. With older versions setting ``type`` to ``"json"`` or ``"text"`` is not supported. @@ -72,11 +86,13 @@ class Selector(_ParselSelector, object_ref): f"{self.__class__.__name__}.__init__() received both response and text" ) - # A response that is neither HTML nor XML, e.g. a JSON one, keeps type - # unset, so that parsel determines it from the body. + # Any other response, e.g. a plain-text one, keeps type unset, so that + # parsel determines it from the body. if type is None: if isinstance(response, XmlResponse): type = "xml" # noqa: A001 + elif isinstance(response, JsonResponse): + type = "json" # noqa: A001 elif response is None or isinstance(response, HtmlResponse): type = "html" # noqa: A001 diff --git a/tests/test_selector.py b/tests/test_selector.py index 98595ac93..ce6de9524 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -4,7 +4,7 @@ import parsel import pytest from packaging import version -from scrapy.http import HtmlResponse, TextResponse, XmlResponse +from scrapy.http import HtmlResponse, JsonResponse, TextResponse, XmlResponse from scrapy.selector import Selector PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) @@ -60,6 +60,30 @@ class TestSelector: '

Hello

' ] + @pytest.mark.skipif(not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support json") + def test_flavor_detection_json(self) -> None: + response = JsonResponse( + "http://example.com", body=b'{"a": "b"}', encoding="utf-8" + ) + assert Selector(response).type == "json" + assert response.jmespath("a").get() == "b" + + @pytest.mark.skipif(not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support json") + def test_flavor_detection_json_with_html_body(self) -> None: + body = b"

Hello

" + response = JsonResponse("http://example.com", body=body, encoding="utf-8") + assert Selector(response).type == "json" + + html_response = response.replace(cls=HtmlResponse) + assert Selector(html_response).type == "html" + assert html_response.css("p::text").get() == "Hello" + + def test_flavor_detection_text(self) -> None: + response = TextResponse( + "http://example.com", body=b"

Hello

", encoding="utf-8" + ) + assert Selector(response).type == "html" + def test_http_header_encoding_precedence(self): # '\xa3' = pound symbol in unicode # '\xc2\xa3' = pound symbol in utf-8