From 1ba0f68483cfb8aa62759e21b3479ec9bea94beb Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 17:09:37 +0200 Subject: [PATCH] Allow comma-separated values in the rel tag Comma-separated `rel` values are often seen in the wild, because Google allows it (see https://developers.google.com/search/docs/advanced/guidelines/qualify-outbound-links). --- scrapy/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 5c986eedc..51cef1e91 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -138,7 +138,7 @@ def md5sum(file): def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" - return rel is not None and 'nofollow' in rel.split() + return rel is not None and 'nofollow' in rel.replace(',', ' ').split() def create_instance(objcls, settings, crawler, *args, **kwargs):