RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix rel_has_nofollow to be case-insensitive per HTML spec (#7632) 

The HTML specification states that link type keywords like 'nofollow'
are ASCII case-insensitive. Sites using rel="NoFollow" or rel="NOFOLLOW"
were incorrectly treated as follow links, causing Scrapy to crawl pages
it should skip.

Fix: add .lower() before the token split so all casing variants of
'nofollow' are correctly recognized.

Co-authored-by: JSap0914 <JSap0914@users.noreply.github.com>
This commit is contained in:
JSap0914 2026-06-17 18:48:43 +09:00 committed by GitHub
parent 0a4a92e843
commit b7824db573
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/utils/misc.py
View File

@ -162,7 +162,7 @@ def md5sum(file: IO[bytes]) -> str:
def rel_has_nofollow(rel: str | None) -> bool:
"""Return True if link rel attribute has nofollow type"""
return rel is not None and "nofollow" in rel.replace(",", " ").split()
return rel is not None and "nofollow" in rel.lower().replace(",", " ").split()
class SupportsFromCrawler(Protocol[_T_co, _P]):

5
tests/test_utils_misc/__init__.py
View File

@ -160,3 +160,8 @@ class TestUtilsMisc:
assert rel_has_nofollow("nofollowfoo") is False
assert rel_has_nofollow("foonofollow") is False
assert rel_has_nofollow("ugc, , nofollow") is True
# rel attribute values are ASCII case-insensitive per the HTML spec
assert rel_has_nofollow("NoFollow") is True
assert rel_has_nofollow("NOFOLLOW") is True
assert rel_has_nofollow("UGC NoFollow") is True
assert rel_has_nofollow("ugc,NoFollow") is True