diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index f40a36d31..713a94e10 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -93,6 +93,12 @@ LxmlLinkExtractor Has the same behaviour as ``restrict_xpaths``. :type restrict_css: str or list + :param restrict_text: a single regular expression (or list of regular expressions) + that the link's text must match in order to be extracted. If not + given (or empty), it will match all links. If a list of regular expressions is + given, the link will be extracted if it matches at least one. + :type restrict_text: a regular expression (or list of) + :param tags: a tag or a list of tags to consider when extracting links. Defaults to ``('a', 'area')``. :type tags: str or list diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 97e8c0af1..ebf3cd7d8 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -50,7 +50,7 @@ class FilteringLinkExtractor(object): _csstranslator = HTMLTranslator() def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, - restrict_xpaths, canonicalize, deny_extensions, restrict_css): + restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): self.link_extractor = link_extractor @@ -70,6 +70,8 @@ class FilteringLinkExtractor(object): if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} + self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(restrict_text)] def _link_allowed(self, link): if not _is_valid_url(link.url): @@ -85,6 +87,8 @@ class FilteringLinkExtractor(object): return False if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): return False + if self.restrict_text and not _matches(link.text, self.restrict_text): + return False return True def matches(self, url): diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index a7092f9b8..8f6f93a44 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -97,7 +97,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), - strip=True): + strip=True, restrict_text=None): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs @@ -111,9 +111,10 @@ class LxmlLinkExtractor(FilteringLinkExtractor): ) super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions) + allow_domains=allow_domains, deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, + canonicalize=canonicalize, deny_extensions=deny_extensions, + restrict_text=restrict_text) def extract_links(self, response): base_url = get_base_url(response) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 5fa6b771c..8940a4d77 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -113,7 +113,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), - strip=True): + strip=True, restrict_text=()): warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " "Please use scrapy.linkextractors.LinkExtractor", @@ -127,13 +127,14 @@ class SgmlLinkExtractor(FilteringLinkExtractor): with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process_value=process_value, strip=strip, - canonicalized=canonicalize) + unique=unique, process_value=process_value, strip=strip, + canonicalized=canonicalize) super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions) + allow_domains=allow_domains, deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, + canonicalize=canonicalize, deny_extensions=deny_extensions, + restrict_text=restrict_text) def extract_links(self, response): base_url = None diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 903032b52..c9cd629f4 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -479,6 +479,30 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) + def test_link_restrict_text(self): + html = b""" + Pic of a cat + Pic of a dog + Pic of a cow + """ + response = HtmlResponse("http://example.org/index.html", body=html) + # Simple text inclusion test + lx = self.extractor_cls(restrict_text='dog') + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + ]) + # Unique regex test + lx = self.extractor_cls(restrict_text=r'of.*dog') + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + ]) + # Multiple regex test + lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat']) + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False), + Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + ]) + @pytest.mark.xfail def test_restrict_xpaths_with_html_entities(self): super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()