RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix arbitrary callable execution from the Referrer-Policy response header

This commit is contained in:
Adrian Chaves 2026-01-19 16:44:03 +01:00
parent 0b9d8da09d
commit b6e5c58ae7
5 changed files with 220 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
docs/news.rst
View File

@ -3,6 +3,26 @@
Release notes
=============
.. _release-2.14.2:
Scrapy 2.14.2 (unreleased)
--------------------------
Security bug fixes
~~~~~~~~~~~~~~~~~~
- Values from the ``Referrer-Policy`` header of HTTP responses are no longer
executed as Python callables. See the `cwxj-rr6w-m6w7`_ security advisory
for details.
.. _cwxj-rr6w-m6w7: https://github.com/scrapy/scrapy/security/advisories/GHSA-cwxj-rr6w-m6w7
New features
~~~~~~~~~~~~
- Added a new setting, :setting:`REFERER_POLICIES`, to allow customizing
supported referrer policies.
.. _release-2.14.1:
Scrapy 2.14.1 (2026-01-12)

17
docs/topics/spider-middleware.rst
View File

@ -403,6 +403,23 @@ String value Class name (as a string)
.. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin
.. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url
.. setting:: REFERRER_POLICIES
REFERRER_POLICIES
^^^^^^^^^^^^^^^^^
Default: ``{}``
A dictionary mapping policy names to import paths of
:class:`scrapy.spidermiddlewares.referer.ReferrerPolicy` subclasses, or
``None`` to disable support for a given policy name.
This allows to override the policies triggered by the ``Referrer-Policy``
response header.
Use ``""`` to override the policy for responses with `no referrer policy
<https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string>`__.
StartSpiderMiddleware
---------------------

112
scrapy/spidermiddlewares/referer.py
View File

@ -17,7 +17,7 @@ from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_unicode
from scrapy.utils.python import _looks_like_import_path, to_unicode
from scrapy.utils.url import strip_url
if TYPE_CHECKING:
@ -287,56 +287,38 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
name: str = POLICY_SCRAPY_DEFAULT
_policy_classes: dict[str, type[ReferrerPolicy]] = {
p.name: p
for p in (
NoReferrerPolicy,
NoReferrerWhenDowngradePolicy,
SameOriginPolicy,
OriginPolicy,
StrictOriginPolicy,
OriginWhenCrossOriginPolicy,
StrictOriginWhenCrossOriginPolicy,
UnsafeUrlPolicy,
DefaultReferrerPolicy,
)
}
# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
_policy_classes[""] = NoReferrerWhenDowngradePolicy
def _load_policy_class(
policy: str, warning_only: bool = False
) -> type[ReferrerPolicy] | None:
"""
Expect a string for the path to the policy class,
otherwise try to interpret the string as a standard value
from https://www.w3.org/TR/referrer-policy/#referrer-policies
"""
try:
return cast("type[ReferrerPolicy]", load_object(policy))
except ValueError:
tokens = [token.strip() for token in policy.lower().split(",")]
# https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header
for token in tokens[::-1]:
if token in _policy_classes:
return _policy_classes[token]
msg = f"Could not load referrer policy {policy!r}"
if not warning_only:
raise RuntimeError(msg)
warnings.warn(msg, RuntimeWarning)
return None
class RefererMiddleware(BaseSpiderMiddleware):
def __init__(self, settings: BaseSettings | None = None): # pylint: disable=super-init-not-called
self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy
if settings is not None:
settings_policy = _load_policy_class(settings.get("REFERRER_POLICY"))
assert settings_policy
self.default_policy = settings_policy
self.policies: dict[str, type[ReferrerPolicy]] = {
p.name: p
for p in (
NoReferrerPolicy,
NoReferrerWhenDowngradePolicy,
SameOriginPolicy,
OriginPolicy,
StrictOriginPolicy,
OriginWhenCrossOriginPolicy,
StrictOriginWhenCrossOriginPolicy,
UnsafeUrlPolicy,
DefaultReferrerPolicy,
)
}
# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
self.policies[""] = NoReferrerWhenDowngradePolicy
if settings is None:
return
setting_policies = settings.getdict("REFERRER_POLICIES")
for policy_name, policy_class_import_path in setting_policies.items():
if policy_class_import_path is None:
del self.policies[policy_name]
else:
self.policies[policy_name] = load_object(policy_class_import_path)
settings_policy = self._load_policy_class(
settings.get("REFERRER_POLICY"), allow_import_path=True
)
assert settings_policy
self.default_policy = settings_policy
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
@ -362,17 +344,47 @@ class RefererMiddleware(BaseSpiderMiddleware):
it is used if valid
- otherwise, the policy from settings is used.
"""
allow_import_path = True
policy_name = request.meta.get("referrer_policy")
if policy_name is None and isinstance(resp_or_url, Response):
policy_header = resp_or_url.headers.get("Referrer-Policy")
if policy_header is not None:
policy_name = to_unicode(policy_header.decode("latin1"))
allow_import_path = False
if policy_name is None:
return self.default_policy()
cls = _load_policy_class(policy_name, warning_only=True)
cls = self._load_policy_class(
policy_name, warning_only=True, allow_import_path=allow_import_path
)
return cls() if cls else self.default_policy()
def _load_policy_class(
self,
policy: str,
warning_only: bool = False,
*,
allow_import_path: bool = False,
) -> type[ReferrerPolicy] | None:
if allow_import_path:
try:
return cast("type[ReferrerPolicy]", load_object(policy))
except ValueError:
pass
policy_names = [
policy_name.strip() for policy_name in policy.lower().split(",")
]
# https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header
for policy_name in policy_names[::-1]:
if policy_name in self.policies:
return self.policies[policy_name]
msg = f"Could not load referrer policy {policy!r}"
if not allow_import_path and _looks_like_import_path(policy):
msg += " (import paths from the response Referrer-Policy header are not allowed)"
if not warning_only:
raise RuntimeError(msg)
warnings.warn(msg, RuntimeWarning)
return None
def get_processed_request(
self, request: Request, response: Response | None
) -> Request | None:

20
scrapy/utils/python.py
View File

@ -335,3 +335,23 @@ class MutableAsyncChain(AsyncIterator[_T]):
async def __anext__(self) -> _T:
return await self.data.__anext__()
def _looks_like_import_path(value: str) -> bool:
"""Return True if **value** looks like a valid Python import path or False
otherwise."""
if not value:
return False
if any(c.isspace() for c in value):
return False
allowed_chars = set(
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_."
)
if any(c not in allowed_chars for c in value):
return False
if value[0] == "." or value[-1] == ".":
return False
parts = value.split(".")
if any(part == "" for part in parts):
return False
return all(part.isidentifier() for part in parts)

101
tests/test_spidermiddleware_referer.py
View File

@ -31,6 +31,8 @@ from scrapy.spidermiddlewares.referer import (
StrictOriginWhenCrossOriginPolicy,
UnsafeUrlPolicy,
)
from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@ -1358,3 +1360,102 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
None,
),
]
@deferred_f_from_coro_f
async def test_response_policy_only_supports_policy_names():
crawler = get_crawler(settings_dict={"REFERRER_POLICY": "no-referrer"})
mw = build_from_crawler(RefererMiddleware, crawler)
async def input_result():
yield Request("https://example.com/")
response = Response(
"https://example.com/",
headers={
"Referrer-Policy": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
},
)
with pytest.warns(
RuntimeWarning,
match=r"Could not load referrer policy 'scrapy\.spidermiddlewares\.referer\.NoReferrerWhenDowngradePolicy' \(import paths from the response Referrer-Policy header are not allowed\)",
):
output = [
request
async for request in mw.process_spider_output_async(
response, input_result()
)
]
assert len(output) == 1
assert b"Referer" not in output[0].headers
response = Response(
"https://example.com/",
headers={"Referrer-Policy": "no-referrer-when-downgrade"},
)
output = [
request
async for request in mw.process_spider_output_async(response, input_result())
]
assert len(output) == 1
assert output[0].headers == {b"Referer": [b"https://example.com/"]}
@deferred_f_from_coro_f
async def test_referer_policies_setting():
crawler = get_crawler(
settings_dict={
"REFERRER_POLICY": "no-referrer",
"REFERRER_POLICIES": {
"no-referrer-when-downgrade": None,
"custom-policy": CustomPythonOrgPolicy,
"": CustomPythonOrgPolicy,
},
}
)
mw = build_from_crawler(RefererMiddleware, crawler)
async def input_result():
yield Request("https://example.com/")
# "no-referrer-when-downgrade": None,
response = Response(
"https://example.com/",
headers={"Referrer-Policy": "no-referrer-when-downgrade"},
)
with pytest.warns(
RuntimeWarning,
match=r"Could not load referrer policy 'no-referrer-when-downgrade'",
):
output = [
request
async for request in mw.process_spider_output_async(
response, input_result()
)
]
assert len(output) == 1
assert b"Referer" not in output[0].headers
# "custom-policy": CustomPythonOrgPolicy,
response = Response(
"https://example.com/",
headers={"Referrer-Policy": "custom-policy"},
)
output = [
request
async for request in mw.process_spider_output_async(response, input_result())
]
assert len(output) == 1
assert output[0].headers == {b"Referer": [b"https://python.org/"]}
# "": CustomPythonOrgPolicy,
response = Response(
"https://example.com/",
headers={"Referrer-Policy": ""},
)
output = [
request
async for request in mw.process_spider_output_async(response, input_result())
]
assert len(output) == 1
assert output[0].headers == {b"Referer": [b"https://python.org/"]}