LxmlLinkExtractor: add deny_attrs and deny_tags (#7679)

docs/topics/link-extractors.rst

@@ -47,108 +47,7 @@ LxmlLinkExtractor
    :synopsis: lxml's HTMLParser-based link extractors
 
 
-.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, strip=True)
+.. autoclass:: LxmlLinkExtractor
-
-    LxmlLinkExtractor is the recommended link extractor with handy filtering
-    options. It is implemented using lxml's robust HTMLParser.
-
-    :param allow: a single regular expression (or list of regular expressions)
-        that the (absolute) urls must match in order to be extracted. If not
-        given (or empty), it will match all links.
-    :type allow: str or list
-
-    :param deny: a single regular expression (or list of regular expressions)
-        that the (absolute) urls must match in order to be excluded (i.e. not
-        extracted). It has precedence over the ``allow`` parameter. If not
-        given (or empty) it won't exclude any links.
-    :type deny: str or list
-
-    :param allow_domains: a single value or a list of string containing
-        domains which will be considered for extracting the links
-    :type allow_domains: str or list
-
-    :param deny_domains: a single value or a list of strings containing
-        domains which won't be considered for extracting the links
-    :type deny_domains: str or list
-
-    :param deny_extensions: a single value or list of strings containing
-        extensions that should be ignored when extracting links.
-        If not given, it will default to
-        :data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
-
-    :type deny_extensions: list
-
-    :param restrict_xpaths: is an XPath (or list of XPath's) which defines
-        regions inside the response where links should be extracted from.
-        If given, only the text selected by those XPath will be scanned for
-        links.
-    :type restrict_xpaths: str or list
-
-    :param restrict_css: a CSS selector (or list of selectors) which defines
-        regions inside the response where links should be extracted from.
-        Has the same behaviour as ``restrict_xpaths``.
-    :type restrict_css: str or list
-
-    :param restrict_text: a single regular expression (or list of regular expressions)
-        that the link's text must match in order to be extracted. If not
-        given (or empty), it will match all links. If a list of regular expressions is
-        given, the link will be extracted if it matches at least one.
-    :type restrict_text: str or list
-
-    :param tags: a tag or a list of tags to consider when extracting links.
-        Defaults to ``('a', 'area')``.
-    :type tags: str or list
-
-    :param attrs: an attribute or list of attributes which should be considered when looking
-        for links to extract (only for those tags specified in the ``tags``
-        parameter). Defaults to ``('href',)``
-    :type attrs: list
-
-    :param canonicalize: canonicalize each extracted url (using
-        w3lib.url.canonicalize_url). Defaults to ``False``.
-        Note that canonicalize_url is meant for duplicate checking;
-        it can change the URL visible at server side, so the response can be
-        different for requests with canonicalized and raw URLs. If you're
-        using LinkExtractor to follow links it is more robust to
-        keep the default ``canonicalize=False``.
-    :type canonicalize: bool
-
-    :param unique: whether duplicate filtering should be applied to extracted
-        links.
-    :type unique: bool
-
-    :param process_value: a function which receives each value extracted from
-        the tag and attributes scanned and can modify the value and return a
-        new one, or return ``None`` to ignore the link altogether. If not
-        given, ``process_value`` defaults to ``lambda x: x``.
-
-        .. highlight:: html
-
-        For example, to extract links from this code::
-
-            <a href="javascript:goToPage('../other/page.html'); return false">Link text</a>
-
-        .. highlight:: python
-
-        You can use the following function in ``process_value``:
-
-        .. code-block:: python
-
-            def process_value(value):
-                m = re.search(r"javascript:goToPage\('(.*?)'", value)
-                if m:
-                    return m.group(1)
-
-    :type process_value: collections.abc.Callable
-
-    :param strip: whether to strip whitespaces from extracted attributes.
-        According to HTML5 standard, leading and trailing whitespaces
-        must be stripped from ``href`` attributes of ``<a>``, ``<area>``
-        and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
-        elements, etc., so LinkExtractor strips space chars by default.
-        Set ``strip=False`` to turn it off (e.g. if you're extracting urls
-        from elements or attributes which allow leading/trailing whitespaces).
-    :type strip: bool
 
     .. automethod:: extract_links
 

scrapy/linkextractors/lxmlhtml.py

@@ -57,6 +57,18 @@ def _canonicalize_link_url(link: Link) -> str:
     return canonicalize_url(link.url, keep_fragments=True)
 
 
+def _name_matches(allowed: set[str], denied: set[str], name: str) -> bool:
+    """Return whether a tag or attribute *name* should be considered.
+
+    A name matches when it is allowed and not denied. A name is allowed if it is
+    listed in *allowed*, or if *allowed* contains the ``"*"`` wildcard, which
+    matches every name. *denied* has precedence over *allowed*.
+    """
+    if name in denied:
+        return False
+    return "*" in allowed or name in allowed
+
+
 class LxmlParserLinkExtractor:
     def __init__(
         self,
@@ -162,6 +174,126 @@ _RegexOrSeveral: TypeAlias = _Regex | Iterable[_Regex]
 
 
 class LxmlLinkExtractor:
+    r"""LxmlLinkExtractor is the recommended link extractor with handy filtering
+    options. It is implemented using lxml's robust HTMLParser.
+
+    :param allow: a single regular expression (or list of regular expressions)
+        that the (absolute) urls must match in order to be extracted. If not
+        given (or empty), it will match all links.
+    :type allow: str or list
+
+    :param deny: a single regular expression (or list of regular expressions)
+        that the (absolute) urls must match in order to be excluded (i.e. not
+        extracted). It has precedence over the ``allow`` parameter. If not
+        given (or empty) it won't exclude any links.
+    :type deny: str or list
+
+    :param allow_domains: a single value or a list of string containing
+        domains which will be considered for extracting the links
+    :type allow_domains: str or list
+
+    :param deny_domains: a single value or a list of strings containing
+        domains which won't be considered for extracting the links
+    :type deny_domains: str or list
+
+    :param deny_extensions: a single value or list of strings containing
+        extensions that should be ignored when extracting links.
+        If not given, it will default to
+        :data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
+    :type deny_extensions: list
+
+    :param restrict_xpaths: is an XPath (or list of XPath's) which defines
+        regions inside the response where links should be extracted from.
+        If given, only the text selected by those XPath will be scanned for
+        links.
+    :type restrict_xpaths: str or list
+
+    :param restrict_css: a CSS selector (or list of selectors) which defines
+        regions inside the response where links should be extracted from.
+        Has the same behaviour as ``restrict_xpaths``.
+    :type restrict_css: str or list
+
+    :param restrict_text: a single regular expression (or list of regular
+        expressions) that the link's text must match in order to be extracted.
+        If not given (or empty), it will match all links. If a list of regular
+        expressions is given, the link will be extracted if it matches at least
+        one.
+    :type restrict_text: str or list
+
+    :param tags: a tag or a list of tags to consider when extracting links.
+        Defaults to ``('a', 'area')``. Use ``'*'`` to consider every tag.
+    :type tags: str or list
+
+    :param attrs: an attribute or list of attributes which should be considered
+        when looking for links to extract (only for those tags specified in the
+        ``tags`` parameter). Defaults to ``('href',)``. Use ``'*'`` to consider
+        every attribute.
+    :type attrs: list
+
+    :param deny_tags: a tag or a list of tags that should not be considered when
+        extracting links. It has precedence over the ``tags`` parameter, so it
+        can be combined with ``tags='*'`` to consider every tag except a few.
+        Defaults to ``()`` (no tag is excluded).
+
+        .. versionadded:: VERSION
+    :type deny_tags: str or list
+
+    :param deny_attrs: an attribute or a list of attributes that should not be
+        considered when looking for links to extract. It has precedence over the
+        ``attrs`` parameter, so it can be combined with ``attrs='*'`` to consider
+        every attribute except a few. Defaults to ``()`` (no attribute is
+        excluded).
+
+        .. versionadded:: VERSION
+    :type deny_attrs: str or list
+
+    :param canonicalize: canonicalize each extracted url (using
+        w3lib.url.canonicalize_url). Defaults to ``False``.
+        Note that canonicalize_url is meant for duplicate checking;
+        it can change the URL visible at server side, so the response can be
+        different for requests with canonicalized and raw URLs. If you're
+        using LinkExtractor to follow links it is more robust to
+        keep the default ``canonicalize=False``.
+    :type canonicalize: bool
+
+    :param unique: whether duplicate filtering should be applied to extracted
+        links.
+    :type unique: bool
+
+    :param process_value: a function which receives each value extracted from
+        the tag and attributes scanned and can modify the value and return a
+        new one, or return ``None`` to ignore the link altogether. If not
+        given, ``process_value`` defaults to ``lambda x: x``.
+
+        .. highlight:: html
+
+        For example, to extract links from this code::
+
+            <a href="javascript:goToPage('../other/page.html'); return false">Link text</a>
+
+        .. highlight:: python
+
+        You can use the following function in ``process_value``:
+
+        .. code-block:: python
+
+            def process_value(value):
+                m = re.search(r"javascript:goToPage\('(.*?)'", value)
+                if m:
+                    return m.group(1)
+
+    :type process_value: collections.abc.Callable
+
+    :param strip: whether to strip whitespaces from extracted attributes.
+        According to HTML5 standard, leading and trailing whitespaces
+        must be stripped from ``href`` attributes of ``<a>``, ``<area>``
+        and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
+        elements, etc., so LinkExtractor strips space chars by default.
+        Set ``strip=False`` to turn it off (e.g. if you're extracting urls
+        from elements or attributes which allow leading/trailing whitespaces).
+    :type strip: bool
+    """
+
     _csstranslator = HTMLTranslator()
 
     def __init__(
@@ -180,11 +312,17 @@ class LxmlLinkExtractor:
         restrict_css: str | Iterable[str] = (),
         strip: bool = True,
         restrict_text: _RegexOrSeveral | None = None,
+        deny_tags: str | Iterable[str] = (),
+        deny_attrs: str | Iterable[str] = (),
     ):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
+        deny_tags, deny_attrs = (
+            set(arg_to_iter(deny_tags)),
+            set(arg_to_iter(deny_attrs)),
+        )
         self.link_extractor = LxmlParserLinkExtractor(
-            tag=partial(operator.contains, tags),
+            tag=partial(_name_matches, tags, deny_tags),
-            attr=partial(operator.contains, attrs),
+            attr=partial(_name_matches, attrs, deny_attrs),
             unique=unique,
             process=process_value,
             strip=strip,

tests/test_linkextractors.py

@@ -499,6 +499,59 @@ class Base:
                 ),
             ]
 
+        def test_tags_attrs_wildcard_and_deny(self):
+            html = b"""
+            <html><body>
+            <a href="a.html">a</a>
+            <div data-url="div.html">div</div>
+            <span href="span.html">span</span>
+            <p data-url="p.html">p</p>
+            </body></html>
+            """
+            response = HtmlResponse("http://example.com/index.html", body=html)
+
+            def urls(**kwargs):
+                lx = self.extractor_cls(**kwargs)
+                return [link.url for link in lx.extract_links(response)]
+
+            # Default behavior is unchanged: only the listed tags and attributes.
+            assert urls() == ["http://example.com/a.html"]
+
+            # "*" as a tag matches every tag.
+            assert urls(tags="*") == [
+                "http://example.com/a.html",
+                "http://example.com/span.html",
+            ]
+
+            # "*" as an attribute matches every attribute.
+            assert urls(tags="*", attrs="*") == [
+                "http://example.com/a.html",
+                "http://example.com/div.html",
+                "http://example.com/span.html",
+                "http://example.com/p.html",
+            ]
+
+            # deny_tags excludes tags from the wildcard.
+            assert urls(tags="*", attrs="*", deny_tags="div") == [
+                "http://example.com/a.html",
+                "http://example.com/span.html",
+                "http://example.com/p.html",
+            ]
+
+            # deny_attrs excludes attributes from the wildcard.
+            assert urls(tags="*", attrs="*", deny_attrs="data-url") == [
+                "http://example.com/a.html",
+                "http://example.com/span.html",
+            ]
+
+            # deny_tags also applies when tags are listed explicitly.
+            assert urls(tags=("a", "span"), attrs="*", deny_tags="span") == [
+                "http://example.com/a.html",
+            ]
+
+            # The wildcard for one parameter is independent of the other.
+            assert urls(tags="a", attrs="*") == ["http://example.com/a.html"]
+
         def test_xhtml(self):
             xhtml = b"""
     <?xml version="1.0"?>