diff --git a/docs/news.rst b/docs/news.rst index a7d8e50f3..ee9d7c017 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -11,6 +11,12 @@ Scrapy 2.14.2 (unreleased) Security bug fixes ~~~~~~~~~~~~~~~~~~ +- Values from the ``Referrer-Policy`` header of HTTP responses are no longer + executed as Python callables. See the `cwxj-rr6w-m6w7`_ security advisory + for details. + + .. _cwxj-rr6w-m6w7: https://github.com/scrapy/scrapy/security/advisories/GHSA-cwxj-rr6w-m6w7 + - In line with the `standard `__, 301 redirects of ``POST`` requests turn into ``GET`` requests. @@ -30,6 +36,12 @@ Deprecations The parameter has also been renamed to ``response`` to reflect this change. The old parameter name (``resp_or_url``) is deprecated. +New features +~~~~~~~~~~~~ + +- Added a new setting, :setting:`REFERER_POLICIES`, to allow customizing + supported referrer policies. + Bug fixes ~~~~~~~~~ diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 799d2f99c..8b000697a 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -403,6 +403,25 @@ String value Class name (as a string) .. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin .. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url +.. setting:: REFERRER_POLICIES + +REFERRER_POLICIES +^^^^^^^^^^^^^^^^^ + +.. versionadded:: 2.14.2 + +Default: ``{}`` + +A dictionary mapping policy names to import paths of +:class:`scrapy.spidermiddlewares.referer.ReferrerPolicy` subclasses, or +``None`` to disable support for a given policy name. + +This allows overriding the policies triggered by the ``Referrer-Policy`` +response header. + +Use ``""`` to override the policy for responses with `no referrer policy +`__. + StartSpiderMiddleware --------------------- diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 5825f6467..4251a9e7b 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -11,14 +11,11 @@ from typing import TYPE_CHECKING, cast from urllib.parse import urlparse from warnings import warn -from w3lib.url import safe_url_string - -from scrapy import Spider, signals from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.spidermiddlewares.base import BaseSpiderMiddleware from scrapy.utils.misc import load_object -from scrapy.utils.python import to_unicode +from scrapy.utils.python import _looks_like_import_path, to_unicode from scrapy.utils.url import strip_url if TYPE_CHECKING: @@ -288,56 +285,38 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): name: str = POLICY_SCRAPY_DEFAULT -_policy_classes: dict[str, type[ReferrerPolicy]] = { - p.name: p - for p in ( - NoReferrerPolicy, - NoReferrerWhenDowngradePolicy, - SameOriginPolicy, - OriginPolicy, - StrictOriginPolicy, - OriginWhenCrossOriginPolicy, - StrictOriginWhenCrossOriginPolicy, - UnsafeUrlPolicy, - DefaultReferrerPolicy, - ) -} - -# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string -_policy_classes[""] = NoReferrerWhenDowngradePolicy - - -def _load_policy_class( - policy: str, warning_only: bool = False -) -> type[ReferrerPolicy] | None: - """ - Expect a string for the path to the policy class, - otherwise try to interpret the string as a standard value - from https://www.w3.org/TR/referrer-policy/#referrer-policies - """ - try: - return cast("type[ReferrerPolicy]", load_object(policy)) - except ValueError: - tokens = [token.strip() for token in policy.lower().split(",")] - # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header - for token in tokens[::-1]: - if token in _policy_classes: - return _policy_classes[token] - - msg = f"Could not load referrer policy {policy!r}" - if not warning_only: - raise RuntimeError(msg) - warnings.warn(msg, RuntimeWarning) - return None - - class RefererMiddleware(BaseSpiderMiddleware): def __init__(self, settings: BaseSettings | None = None): # pylint: disable=super-init-not-called self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy - if settings is not None: - settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) - assert settings_policy - self.default_policy = settings_policy + self.policies: dict[str, type[ReferrerPolicy]] = { + p.name: p + for p in ( + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + SameOriginPolicy, + OriginPolicy, + StrictOriginPolicy, + OriginWhenCrossOriginPolicy, + StrictOriginWhenCrossOriginPolicy, + UnsafeUrlPolicy, + DefaultReferrerPolicy, + ) + } + # Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string + self.policies[""] = NoReferrerWhenDowngradePolicy + if settings is None: + return + setting_policies = settings.getdict("REFERRER_POLICIES") + for policy_name, policy_class_import_path in setting_policies.items(): + if policy_class_import_path is None: + del self.policies[policy_name] + else: + self.policies[policy_name] = load_object(policy_class_import_path) + settings_policy = self._load_policy_class( + settings.get("REFERRER_POLICY"), allow_import_path=True + ) + assert settings_policy + self.default_policy = settings_policy @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -381,17 +360,70 @@ class RefererMiddleware(BaseSpiderMiddleware): DeprecationWarning, stacklevel=2, ) + allow_import_path = True policy_name = request.meta.get("referrer_policy") if policy_name is None and isinstance(response, Response): policy_header = response.headers.get("Referrer-Policy") if policy_header is not None: policy_name = to_unicode(policy_header.decode("latin1")) + allow_import_path = False if policy_name is None: return self.default_policy() - - cls = _load_policy_class(policy_name, warning_only=True) + cls = self._load_policy_class( + policy_name, warning_only=True, allow_import_path=allow_import_path + ) return cls() if cls else self.default_policy() + def _load_policy_class( + self, + policy: str, + warning_only: bool = False, + *, + allow_import_path: bool = False, + ) -> type[ReferrerPolicy] | None: + """Load the :class:`ReferrerPolicy` class to use for *policy*. + + *policy* may be any of the following: + + - A standard policy name, e.g. ``"no-referrer"``, + ``"origin-when-cross-origin"``, etc. + + - The special ``"scrapy-default"`` policy. + + - The import path of a :class:`ReferrerPolicy` subclass, e.g. + ``"scrapy.spidermiddlewares.referer.NoReferrerPolicy"`` or + ``"myproject.policies.CustomReferrerPolicy"``. + + If *warning_only* is ``False`` (default) and *policy* cannot be turned + into a :class:`ReferrerPolicy` subclass, a :exc:`RuntimeError` is + raised. If *warning_only* is ``True``, a warning is logged and ``None`` + is returned instead. + + If *allow_import_path* is ``False`` (default), import paths are not + allowed, resulting in :exc:`RuntimeError` or ``None``. If ``True``, + they are allowed. Use ``True`` only if you trust the source of the + *policy* value. + """ + if allow_import_path: + try: + return cast("type[ReferrerPolicy]", load_object(policy)) + except ValueError: + pass + policy_names = [ + policy_name.strip() for policy_name in policy.lower().split(",") + ] + # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header + for policy_name in policy_names[::-1]: + if policy_name in self.policies: + return self.policies[policy_name] + msg = f"Could not load referrer policy {policy!r}" + if not allow_import_path and _looks_like_import_path(policy): + msg += " (import paths from the response Referrer-Policy header are not allowed)" + if not warning_only: + raise RuntimeError(msg) + warnings.warn(msg, RuntimeWarning) + return None + def get_processed_request( self, request: Request, response: Response | None ) -> Request | None: diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 04da75a30..bf70fa7fc 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -335,3 +335,23 @@ class MutableAsyncChain(AsyncIterator[_T]): async def __anext__(self) -> _T: return await self.data.__anext__() + + +def _looks_like_import_path(value: str) -> bool: + """Return True if **value** looks like a valid Python import path or False + otherwise.""" + if not value: + return False + if any(c.isspace() for c in value): + return False + allowed_chars = set( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_." + ) + if any(c not in allowed_chars for c in value): + return False + if value[0] == "." or value[-1] == ".": + return False + parts = value.split(".") + if any(part == "" for part in parts): + return False + return all(part.isidentifier() for part in parts) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b4090b78c..444839019 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, cast +from typing import Any from urllib.parse import urlparse import pytest @@ -30,15 +30,10 @@ from scrapy.spidermiddlewares.referer import ( StrictOriginWhenCrossOriginPolicy, UnsafeUrlPolicy, ) +from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.misc import build_from_crawler -from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -if TYPE_CHECKING: - from collections.abc import Sequence - - from scrapy.crawler import Crawler - class TestRefererMiddleware: req_meta: dict[str, Any] = {} @@ -998,7 +993,9 @@ class TestPolicyMethodResponseParamRename: self.mw.policy(resp_or_url=self.response, request=self.request) found = False for warning in w: - if "Passing 'resp_or_url' is deprecated, use 'response' instead" in str(warning.message): + if "Passing 'resp_or_url' is deprecated, use 'response' instead" in str( + warning.message + ): found = True break assert found @@ -1010,7 +1007,7 @@ class TestPolicyMethodResponseParamRename: assert "resp_or_url" not in str(warning.message) def test_key_response_string(self): - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") self.mw.policy(response="http://old.com", request=self.request) found = False @@ -1021,7 +1018,108 @@ class TestPolicyMethodResponseParamRename: assert found def test_both_resp_or_url_and_response(self): - with pytest.raises(TypeError, match="Cannot pass both 'response' and 'resp_or_url'"): - self.mw.policy(response=self.response, resp_or_url=self.response, request=self.request) + with pytest.raises( + TypeError, match="Cannot pass both 'response' and 'resp_or_url'" + ): + self.mw.policy( + response=self.response, resp_or_url=self.response, request=self.request + ) +@deferred_f_from_coro_f +async def test_response_policy_only_supports_policy_names(): + crawler = get_crawler(settings_dict={"REFERRER_POLICY": "no-referrer"}) + mw = build_from_crawler(RefererMiddleware, crawler) + + async def input_result(): + yield Request("https://example.com/") + + response = Response( + "https://example.com/", + headers={ + "Referrer-Policy": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy" + }, + ) + with pytest.warns( + RuntimeWarning, + match=r"Could not load referrer policy 'scrapy\.spidermiddlewares\.referer\.NoReferrerWhenDowngradePolicy' \(import paths from the response Referrer-Policy header are not allowed\)", + ): + output = [ + request + async for request in mw.process_spider_output_async( + response, input_result() + ) + ] + assert len(output) == 1 + assert b"Referer" not in output[0].headers + + response = Response( + "https://example.com/", + headers={"Referrer-Policy": "no-referrer-when-downgrade"}, + ) + output = [ + request + async for request in mw.process_spider_output_async(response, input_result()) + ] + assert len(output) == 1 + assert output[0].headers == {b"Referer": [b"https://example.com/"]} + + +@deferred_f_from_coro_f +async def test_referer_policies_setting(): + crawler = get_crawler( + settings_dict={ + "REFERRER_POLICY": "no-referrer", + "REFERRER_POLICIES": { + "no-referrer-when-downgrade": None, + "custom-policy": CustomPythonOrgPolicy, + "": CustomPythonOrgPolicy, + }, + } + ) + mw = build_from_crawler(RefererMiddleware, crawler) + + async def input_result(): + yield Request("https://example.com/") + + # "no-referrer-when-downgrade": None, + response = Response( + "https://example.com/", + headers={"Referrer-Policy": "no-referrer-when-downgrade"}, + ) + with pytest.warns( + RuntimeWarning, + match=r"Could not load referrer policy 'no-referrer-when-downgrade'", + ): + output = [ + request + async for request in mw.process_spider_output_async( + response, input_result() + ) + ] + assert len(output) == 1 + assert b"Referer" not in output[0].headers + + # "custom-policy": CustomPythonOrgPolicy, + response = Response( + "https://example.com/", + headers={"Referrer-Policy": "custom-policy"}, + ) + output = [ + request + async for request in mw.process_spider_output_async(response, input_result()) + ] + assert len(output) == 1 + assert output[0].headers == {b"Referer": [b"https://python.org/"]} + + # "": CustomPythonOrgPolicy, + response = Response( + "https://example.com/", + headers={"Referrer-Policy": ""}, + ) + output = [ + request + async for request in mw.process_spider_output_async(response, input_result()) + ] + assert len(output) == 1 + assert output[0].headers == {b"Referer": [b"https://python.org/"]}