Merge pull request #3635 from matthieucham/feature-filteringlinkextractor-restrict-text

[MRG+1] Feature filteringlinkextractor restrict text

docs/topics/link-extractors.rst

@@ -93,6 +93,12 @@ LxmlLinkExtractor
         Has the same behaviour as ``restrict_xpaths``.
     :type restrict_css: str or list
 
+    :param restrict_text: a single regular expression (or list of regular expressions)
+        that the link's text must match in order to be extracted. If not
+        given (or empty), it will match all links. If a list of regular expressions is
+        given, the link will be extracted if it matches at least one.
+    :type restrict_text: a regular expression (or list of)
+
     :param tags: a tag or a list of tags to consider when extracting links.
         Defaults to ``('a', 'area')``.
     :type tags: str or list

scrapy/linkextractors/__init__.py

@@ -50,7 +50,7 @@ class FilteringLinkExtractor(object):
     _csstranslator = HTMLTranslator()
 
     def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
-                 restrict_xpaths, canonicalize, deny_extensions, restrict_css):
+                 restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):
 
         self.link_extractor = link_extractor
 
@@ -70,6 +70,8 @@ class FilteringLinkExtractor(object):
         if deny_extensions is None:
             deny_extensions = IGNORED_EXTENSIONS
         self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)}
+        self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x)
+                              for x in arg_to_iter(restrict_text)]
 
     def _link_allowed(self, link):
         if not _is_valid_url(link.url):
@@ -85,6 +87,8 @@ class FilteringLinkExtractor(object):
             return False
         if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions):
             return False
+        if self.restrict_text and not _matches(link.text, self.restrict_text):
+            return False
         return True
 
     def matches(self, url):

scrapy/linkextractors/lxmlhtml.py

@@ -97,7 +97,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
                  tags=('a', 'area'), attrs=('href',), canonicalize=False,
                  unique=True, process_value=None, deny_extensions=None, restrict_css=(),
-                 strip=True):
+                 strip=True, restrict_text=None):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         tag_func = lambda x: x in tags
         attr_func = lambda x: x in attrs
@@ -111,9 +111,10 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
         )
 
         super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
-            allow_domains=allow_domains, deny_domains=deny_domains,
-            restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
-            canonicalize=canonicalize, deny_extensions=deny_extensions)
+                                                allow_domains=allow_domains, deny_domains=deny_domains,
+                                                restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
+                                                canonicalize=canonicalize, deny_extensions=deny_extensions,
+                                                restrict_text=restrict_text)
 
     def extract_links(self, response):
         base_url = get_base_url(response)

scrapy/linkextractors/sgml.py

@@ -113,7 +113,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
                  tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True,
                  process_value=None, deny_extensions=None, restrict_css=(),
-                 strip=True):
+                 strip=True, restrict_text=()):
         warnings.warn(
             "SgmlLinkExtractor is deprecated and will be removed in future releases. "
             "Please use scrapy.linkextractors.LinkExtractor",
@@ -127,13 +127,14 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', ScrapyDeprecationWarning)
             lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
-                unique=unique, process_value=process_value, strip=strip,
-                canonicalized=canonicalize)
+                                       unique=unique, process_value=process_value, strip=strip,
+                                       canonicalized=canonicalize)
 
         super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
-            allow_domains=allow_domains, deny_domains=deny_domains,
-            restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
-            canonicalize=canonicalize, deny_extensions=deny_extensions)
+                                                allow_domains=allow_domains, deny_domains=deny_domains,
+                                                restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
+                                                canonicalize=canonicalize, deny_extensions=deny_extensions,
+                                                restrict_text=restrict_text)
 
     def extract_links(self, response):
         base_url = None

tests/test_linkextractors.py

@@ -479,6 +479,30 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
             Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
         ])
 
+    def test_link_restrict_text(self):
+        html = b"""
+        <a href="http://example.org/item1.html">Pic of a cat</a>
+        <a href="http://example.org/item2.html">Pic of a dog</a>
+        <a href="http://example.org/item3.html">Pic of a cow</a>
+        """
+        response = HtmlResponse("http://example.org/index.html", body=html)
+        # Simple text inclusion test
+        lx = self.extractor_cls(restrict_text='dog')
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+        ])
+        # Unique regex test
+        lx = self.extractor_cls(restrict_text=r'of.*dog')
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+        ])
+        # Multiple regex test
+        lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat'])
+        self.assertEqual([link for link in lx.extract_links(response)], [
+            Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False),
+            Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+        ])
+
     @pytest.mark.xfail
     def test_restrict_xpaths_with_html_entities(self):
         super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()