Fix rel_has_nofollow to be case-insensitive per HTML spec

The HTML specification states that link type keywords like 'nofollow'
are ASCII case-insensitive. Sites using rel="NoFollow" or rel="NOFOLLOW"
were incorrectly treated as follow links, causing Scrapy to crawl pages
it should skip.

Fix: add .lower() before the token split so all casing variants of
'nofollow' are correctly recognized.

scrapy/utils/misc.py

@@ -162,7 +162,7 @@ def md5sum(file: IO[bytes]) -> str:
 
 def rel_has_nofollow(rel: str | None) -> bool:
     """Return True if link rel attribute has nofollow type"""
-    return rel is not None and "nofollow" in rel.replace(",", " ").split()
+    return rel is not None and "nofollow" in rel.lower().replace(",", " ").split()
 
 
 class SupportsFromCrawler(Protocol[_T_co, _P]):

tests/test_utils_misc/__init__.py

@@ -160,3 +160,8 @@ class TestUtilsMisc:
         assert rel_has_nofollow("nofollowfoo") is False
         assert rel_has_nofollow("foonofollow") is False
         assert rel_has_nofollow("ugc,  ,  nofollow") is True
+        # rel attribute values are ASCII case-insensitive per the HTML spec
+        assert rel_has_nofollow("NoFollow") is True
+        assert rel_has_nofollow("NOFOLLOW") is True
+        assert rel_has_nofollow("UGC NoFollow") is True
+        assert rel_has_nofollow("ugc,NoFollow") is True