From b7824db573ca5a50890024f3e2e63c714a62aa20 Mon Sep 17 00:00:00 2001 From: JSap0914 <116227558+JSap0914@users.noreply.github.com> Date: Wed, 17 Jun 2026 18:48:43 +0900 Subject: [PATCH] Fix rel_has_nofollow to be case-insensitive per HTML spec (#7632) The HTML specification states that link type keywords like 'nofollow' are ASCII case-insensitive. Sites using rel="NoFollow" or rel="NOFOLLOW" were incorrectly treated as follow links, causing Scrapy to crawl pages it should skip. Fix: add .lower() before the token split so all casing variants of 'nofollow' are correctly recognized. Co-authored-by: JSap0914 --- scrapy/utils/misc.py | 2 +- tests/test_utils_misc/__init__.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 0b67eaa34..47568e656 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -162,7 +162,7 @@ def md5sum(file: IO[bytes]) -> str: def rel_has_nofollow(rel: str | None) -> bool: """Return True if link rel attribute has nofollow type""" - return rel is not None and "nofollow" in rel.replace(",", " ").split() + return rel is not None and "nofollow" in rel.lower().replace(",", " ").split() class SupportsFromCrawler(Protocol[_T_co, _P]): diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index a995e38e6..c4c861404 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -160,3 +160,8 @@ class TestUtilsMisc: assert rel_has_nofollow("nofollowfoo") is False assert rel_has_nofollow("foonofollow") is False assert rel_has_nofollow("ugc, , nofollow") is True + # rel attribute values are ASCII case-insensitive per the HTML spec + assert rel_has_nofollow("NoFollow") is True + assert rel_has_nofollow("NOFOLLOW") is True + assert rel_has_nofollow("UGC NoFollow") is True + assert rel_has_nofollow("ugc,NoFollow") is True