Merge pull request #5225 from divtiply/patch-3

Allow comma-separated values in the rel attribute

scrapy/utils/misc.py

@@ -138,7 +138,7 @@ def md5sum(file):
 
 def rel_has_nofollow(rel):
     """Return True if link rel attribute has nofollow type"""
-    return rel is not None and 'nofollow' in rel.split()
+    return rel is not None and 'nofollow' in rel.replace(',', ' ').split()
 
 
 def create_instance(objcls, settings, crawler, *args, **kwargs):

tests/test_utils_misc/__init__.py

@@ -4,7 +4,7 @@ import unittest
 from unittest import mock
 
 from scrapy.item import Item, Field
-from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules
+from scrapy.utils.misc import arg_to_iter, create_instance, load_object, rel_has_nofollow, set_environ, walk_modules
 
 
 __doctests__ = ['scrapy.utils.misc']
@@ -162,6 +162,15 @@ class UtilsMiscTestCase(unittest.TestCase):
             assert os.environ.get('some_test_environ') == 'test_value'
         assert os.environ.get('some_test_environ') == 'test'
 
+    def test_rel_has_nofollow(self):
+        assert rel_has_nofollow('ugc nofollow') is True
+        assert rel_has_nofollow('ugc,nofollow') is True
+        assert rel_has_nofollow('ugc') is False
+        assert rel_has_nofollow('nofollow') is True
+        assert rel_has_nofollow('nofollowfoo') is False
+        assert rel_has_nofollow('foonofollow') is False
+        assert rel_has_nofollow('ugc,  ,  nofollow') is True
+
 
 if __name__ == "__main__":
     unittest.main()