RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'cwxj-rr6w-m6w7/fix-referer-policy-handling' into 2.14

This commit is contained in:
Adrian Chaves 2026-03-12 13:37:24 +01:00
parent 8974580e43 04db6a5424
commit 945b787a26
5 changed files with 245 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
docs/news.rst
View File

@ -11,6 +11,12 @@ Scrapy 2.14.2 (unreleased)
Security bug fixes
~~~~~~~~~~~~~~~~~~
- Values from the ``Referrer-Policy`` header of HTTP responses are no longer
executed as Python callables. See the `cwxj-rr6w-m6w7`_ security advisory
for details.
.. _cwxj-rr6w-m6w7: https://github.com/scrapy/scrapy/security/advisories/GHSA-cwxj-rr6w-m6w7
- In line with the `standard
<https://fetch.spec.whatwg.org/#http-redirect-fetch>`__, 301 redirects of
``POST`` requests turn into ``GET`` requests.
@ -30,6 +36,12 @@ Deprecations
The parameter has also been renamed to ``response`` to reflect this change.
The old parameter name (``resp_or_url``) is deprecated.
New features
~~~~~~~~~~~~
- Added a new setting, :setting:`REFERER_POLICIES`, to allow customizing
supported referrer policies.
Bug fixes
~~~~~~~~~

19
docs/topics/spider-middleware.rst
View File

@ -403,6 +403,25 @@ String value Class name (as a string)
.. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin
.. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url
.. setting:: REFERRER_POLICIES
REFERRER_POLICIES
^^^^^^^^^^^^^^^^^
.. versionadded:: 2.14.2
Default: ``{}``
A dictionary mapping policy names to import paths of
:class:`scrapy.spidermiddlewares.referer.ReferrerPolicy` subclasses, or
``None`` to disable support for a given policy name.
This allows overriding the policies triggered by the ``Referrer-Policy``
response header.
Use ``""`` to override the policy for responses with `no referrer policy
<https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string>`__.
StartSpiderMiddleware
---------------------

138
scrapy/spidermiddlewares/referer.py
View File

@ -11,14 +11,11 @@ from typing import TYPE_CHECKING, cast
from urllib.parse import urlparse
from warnings import warn
from w3lib.url import safe_url_string
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_unicode
from scrapy.utils.python import _looks_like_import_path, to_unicode
from scrapy.utils.url import strip_url
if TYPE_CHECKING:
@ -288,56 +285,38 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
name: str = POLICY_SCRAPY_DEFAULT
_policy_classes: dict[str, type[ReferrerPolicy]] = {
p.name: p
for p in (
NoReferrerPolicy,
NoReferrerWhenDowngradePolicy,
SameOriginPolicy,
OriginPolicy,
StrictOriginPolicy,
OriginWhenCrossOriginPolicy,
StrictOriginWhenCrossOriginPolicy,
UnsafeUrlPolicy,
DefaultReferrerPolicy,
)
}
# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
_policy_classes[""] = NoReferrerWhenDowngradePolicy
def _load_policy_class(
policy: str, warning_only: bool = False
) -> type[ReferrerPolicy] | None:
"""
Expect a string for the path to the policy class,
otherwise try to interpret the string as a standard value
from https://www.w3.org/TR/referrer-policy/#referrer-policies
"""
try:
return cast("type[ReferrerPolicy]", load_object(policy))
except ValueError:
tokens = [token.strip() for token in policy.lower().split(",")]
# https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header
for token in tokens[::-1]:
if token in _policy_classes:
return _policy_classes[token]
msg = f"Could not load referrer policy {policy!r}"
if not warning_only:
raise RuntimeError(msg)
warnings.warn(msg, RuntimeWarning)
return None
class RefererMiddleware(BaseSpiderMiddleware):
def __init__(self, settings: BaseSettings | None = None): # pylint: disable=super-init-not-called
self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy
if settings is not None:
settings_policy = _load_policy_class(settings.get("REFERRER_POLICY"))
assert settings_policy
self.default_policy = settings_policy
self.policies: dict[str, type[ReferrerPolicy]] = {
p.name: p
for p in (
NoReferrerPolicy,
NoReferrerWhenDowngradePolicy,
SameOriginPolicy,
OriginPolicy,
StrictOriginPolicy,
OriginWhenCrossOriginPolicy,
StrictOriginWhenCrossOriginPolicy,
UnsafeUrlPolicy,
DefaultReferrerPolicy,
)
}
# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
self.policies[""] = NoReferrerWhenDowngradePolicy
if settings is None:
return
setting_policies = settings.getdict("REFERRER_POLICIES")
for policy_name, policy_class_import_path in setting_policies.items():
if policy_class_import_path is None:
del self.policies[policy_name]
else:
self.policies[policy_name] = load_object(policy_class_import_path)
settings_policy = self._load_policy_class(
settings.get("REFERRER_POLICY"), allow_import_path=True
)
assert settings_policy
self.default_policy = settings_policy
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
@ -381,17 +360,70 @@ class RefererMiddleware(BaseSpiderMiddleware):
DeprecationWarning,
stacklevel=2,
)
allow_import_path = True
policy_name = request.meta.get("referrer_policy")
if policy_name is None and isinstance(response, Response):
policy_header = response.headers.get("Referrer-Policy")
if policy_header is not None:
policy_name = to_unicode(policy_header.decode("latin1"))
allow_import_path = False
if policy_name is None:
return self.default_policy()
cls = _load_policy_class(policy_name, warning_only=True)
cls = self._load_policy_class(
policy_name, warning_only=True, allow_import_path=allow_import_path
)
return cls() if cls else self.default_policy()
def _load_policy_class(
self,
policy: str,
warning_only: bool = False,
*,
allow_import_path: bool = False,
) -> type[ReferrerPolicy] | None:
"""Load the :class:`ReferrerPolicy` class to use for *policy*.
*policy* may be any of the following:
- A standard policy name, e.g. ``"no-referrer"``,
``"origin-when-cross-origin"``, etc.
- The special ``"scrapy-default"`` policy.
- The import path of a :class:`ReferrerPolicy` subclass, e.g.
``"scrapy.spidermiddlewares.referer.NoReferrerPolicy"`` or
``"myproject.policies.CustomReferrerPolicy"``.
If *warning_only* is ``False`` (default) and *policy* cannot be turned
into a :class:`ReferrerPolicy` subclass, a :exc:`RuntimeError` is
raised. If *warning_only* is ``True``, a warning is logged and ``None``
is returned instead.
If *allow_import_path* is ``False`` (default), import paths are not
allowed, resulting in :exc:`RuntimeError` or ``None``. If ``True``,
they are allowed. Use ``True`` only if you trust the source of the
*policy* value.
"""
if allow_import_path:
try:
return cast("type[ReferrerPolicy]", load_object(policy))
except ValueError:
pass
policy_names = [
policy_name.strip() for policy_name in policy.lower().split(",")
]
# https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header
for policy_name in policy_names[::-1]:
if policy_name in self.policies:
return self.policies[policy_name]
msg = f"Could not load referrer policy {policy!r}"
if not allow_import_path and _looks_like_import_path(policy):
msg += " (import paths from the response Referrer-Policy header are not allowed)"
if not warning_only:
raise RuntimeError(msg)
warnings.warn(msg, RuntimeWarning)
return None
def get_processed_request(
self, request: Request, response: Response | None
) -> Request | None:

20
scrapy/utils/python.py
View File

@ -335,3 +335,23 @@ class MutableAsyncChain(AsyncIterator[_T]):
async def __anext__(self) -> _T:
return await self.data.__anext__()
def _looks_like_import_path(value: str) -> bool:
"""Return True if **value** looks like a valid Python import path or False
otherwise."""
if not value:
return False
if any(c.isspace() for c in value):
return False
allowed_chars = set(
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_."
)
if any(c not in allowed_chars for c in value):
return False
if value[0] == "." or value[-1] == ".":
return False
parts = value.split(".")
if any(part == "" for part in parts):
return False
return all(part.isidentifier() for part in parts)

120
tests/test_spidermiddleware_referer.py
View File

@ -1,7 +1,7 @@
from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Any, cast
from typing import Any
from urllib.parse import urlparse
import pytest
@ -30,15 +30,10 @@ from scrapy.spidermiddlewares.referer import (
StrictOriginWhenCrossOriginPolicy,
UnsafeUrlPolicy,
)
from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
from collections.abc import Sequence
from scrapy.crawler import Crawler
class TestRefererMiddleware:
req_meta: dict[str, Any] = {}
@ -998,7 +993,9 @@ class TestPolicyMethodResponseParamRename:
self.mw.policy(resp_or_url=self.response, request=self.request)
found = False
for warning in w:
if "Passing 'resp_or_url' is deprecated, use 'response' instead" in str(warning.message):
if "Passing 'resp_or_url' is deprecated, use 'response' instead" in str(
warning.message
):
found = True
break
assert found
@ -1010,7 +1007,7 @@ class TestPolicyMethodResponseParamRename:
assert "resp_or_url" not in str(warning.message)
def test_key_response_string(self):
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
self.mw.policy(response="http://old.com", request=self.request)
found = False
@ -1021,7 +1018,108 @@ class TestPolicyMethodResponseParamRename:
assert found
def test_both_resp_or_url_and_response(self):
with pytest.raises(TypeError, match="Cannot pass both 'response' and 'resp_or_url'"):
self.mw.policy(response=self.response, resp_or_url=self.response, request=self.request)
with pytest.raises(
TypeError, match="Cannot pass both 'response' and 'resp_or_url'"
):
self.mw.policy(
response=self.response, resp_or_url=self.response, request=self.request
)
@deferred_f_from_coro_f
async def test_response_policy_only_supports_policy_names():
crawler = get_crawler(settings_dict={"REFERRER_POLICY": "no-referrer"})
mw = build_from_crawler(RefererMiddleware, crawler)
async def input_result():
yield Request("https://example.com/")
response = Response(
"https://example.com/",
headers={
"Referrer-Policy": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
},
)
with pytest.warns(
RuntimeWarning,
match=r"Could not load referrer policy 'scrapy\.spidermiddlewares\.referer\.NoReferrerWhenDowngradePolicy' \(import paths from the response Referrer-Policy header are not allowed\)",
):
output = [
request
async for request in mw.process_spider_output_async(
response, input_result()
)
]
assert len(output) == 1
assert b"Referer" not in output[0].headers
response = Response(
"https://example.com/",
headers={"Referrer-Policy": "no-referrer-when-downgrade"},
)
output = [
request
async for request in mw.process_spider_output_async(response, input_result())
]
assert len(output) == 1
assert output[0].headers == {b"Referer": [b"https://example.com/"]}
@deferred_f_from_coro_f
async def test_referer_policies_setting():
crawler = get_crawler(
settings_dict={
"REFERRER_POLICY": "no-referrer",
"REFERRER_POLICIES": {
"no-referrer-when-downgrade": None,
"custom-policy": CustomPythonOrgPolicy,
"": CustomPythonOrgPolicy,
},
}
)
mw = build_from_crawler(RefererMiddleware, crawler)
async def input_result():
yield Request("https://example.com/")
# "no-referrer-when-downgrade": None,
response = Response(
"https://example.com/",
headers={"Referrer-Policy": "no-referrer-when-downgrade"},
)
with pytest.warns(
RuntimeWarning,
match=r"Could not load referrer policy 'no-referrer-when-downgrade'",
):
output = [
request
async for request in mw.process_spider_output_async(
response, input_result()
)
]
assert len(output) == 1
assert b"Referer" not in output[0].headers
# "custom-policy": CustomPythonOrgPolicy,
response = Response(
"https://example.com/",
headers={"Referrer-Policy": "custom-policy"},
)
output = [
request
async for request in mw.process_spider_output_async(response, input_result())
]
assert len(output) == 1
assert output[0].headers == {b"Referer": [b"https://python.org/"]}
# "": CustomPythonOrgPolicy,
response = Response(
"https://example.com/",
headers={"Referrer-Policy": ""},
)
output = [
request
async for request in mw.process_spider_output_async(response, input_result())
]
assert len(output) == 1
assert output[0].headers == {b"Referer": [b"https://python.org/"]}