diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 0b67eaa34..47568e656 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -162,7 +162,7 @@ def md5sum(file: IO[bytes]) -> str: def rel_has_nofollow(rel: str | None) -> bool: """Return True if link rel attribute has nofollow type""" - return rel is not None and "nofollow" in rel.replace(",", " ").split() + return rel is not None and "nofollow" in rel.lower().replace(",", " ").split() class SupportsFromCrawler(Protocol[_T_co, _P]): diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index a995e38e6..c4c861404 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -160,3 +160,8 @@ class TestUtilsMisc: assert rel_has_nofollow("nofollowfoo") is False assert rel_has_nofollow("foonofollow") is False assert rel_has_nofollow("ugc, , nofollow") is True + # rel attribute values are ASCII case-insensitive per the HTML spec + assert rel_has_nofollow("NoFollow") is True + assert rel_has_nofollow("NOFOLLOW") is True + assert rel_has_nofollow("UGC NoFollow") is True + assert rel_has_nofollow("ugc,NoFollow") is True