From 1ba0f68483cfb8aa62759e21b3479ec9bea94beb Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 17:09:37 +0200 Subject: [PATCH 1/5] Allow comma-separated values in the rel tag Comma-separated `rel` values are often seen in the wild, because Google allows it (see https://developers.google.com/search/docs/advanced/guidelines/qualify-outbound-links). --- scrapy/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 5c986eedc..51cef1e91 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -138,7 +138,7 @@ def md5sum(file): def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" - return rel is not None and 'nofollow' in rel.split() + return rel is not None and 'nofollow' in rel.replace(',', ' ').split() def create_instance(objcls, settings, crawler, *args, **kwargs): From 18b6f30a7359d1a798c30888ddd10a1612d8e711 Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 21:13:50 +0200 Subject: [PATCH 2/5] Add test for rel_has_nofollow --- tests/test_utils_misc/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index e95a3a316..67367dbfb 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -4,7 +4,7 @@ import unittest from unittest import mock from scrapy.item import Item, Field -from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules +from scrapy.utils.misc import arg_to_iter, create_instance, load_object, rel_has_nofollow, set_environ, walk_modules __doctests__ = ['scrapy.utils.misc'] @@ -162,6 +162,12 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test_value' assert os.environ.get('some_test_environ') == 'test' + def test_rel_has_nofollow(self): + assert os.environ.get('some_test_environ') is None + asert rel_has_nofollow('ugc nofollow') == True + asert rel_has_nofollow('ugc,nofollow') == True + asert rel_has_nofollow('ugc') == False + if __name__ == "__main__": unittest.main() From 07d20a8ce45ab0cbf61d08214db4963302661257 Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 21:21:43 +0200 Subject: [PATCH 3/5] Fix test_rel_has_nofollow test --- tests/test_utils_misc/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 67367dbfb..b0d7acd12 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -163,10 +163,9 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test' def test_rel_has_nofollow(self): - assert os.environ.get('some_test_environ') is None - asert rel_has_nofollow('ugc nofollow') == True - asert rel_has_nofollow('ugc,nofollow') == True - asert rel_has_nofollow('ugc') == False + assert rel_has_nofollow('ugc nofollow') == True + assert rel_has_nofollow('ugc,nofollow') == True + assert rel_has_nofollow('ugc') == False if __name__ == "__main__": From 295f0e2bf5c352c6ddf27a188af10bf122d1c6b0 Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 21:38:29 +0200 Subject: [PATCH 4/5] Make flake8 happy --- tests/test_utils_misc/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index b0d7acd12..69f593ccd 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -163,9 +163,9 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test' def test_rel_has_nofollow(self): - assert rel_has_nofollow('ugc nofollow') == True - assert rel_has_nofollow('ugc,nofollow') == True - assert rel_has_nofollow('ugc') == False + assert rel_has_nofollow('ugc nofollow') is True + assert rel_has_nofollow('ugc,nofollow') is True + assert rel_has_nofollow('ugc') is False if __name__ == "__main__": From ce9d6c658b21a5d9d9605a2683b7a143f2077dfa Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 22:21:51 +0200 Subject: [PATCH 5/5] Add more rel_has_nofollow tests --- tests/test_utils_misc/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 69f593ccd..47d73a2dd 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -166,6 +166,10 @@ class UtilsMiscTestCase(unittest.TestCase): assert rel_has_nofollow('ugc nofollow') is True assert rel_has_nofollow('ugc,nofollow') is True assert rel_has_nofollow('ugc') is False + assert rel_has_nofollow('nofollow') is True + assert rel_has_nofollow('nofollowfoo') is False + assert rel_has_nofollow('foonofollow') is False + assert rel_has_nofollow('ugc, , nofollow') is True if __name__ == "__main__":