Merge 0b53777d09 into e28e56aa61

docs/topics/link-extractors.rst

@@ -32,7 +32,6 @@ Link extractor reference
 ========================
 
 .. module:: scrapy.linkextractors
-   :synopsis: Link extractors classes
 
 The link extractor class is
 :class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it
@@ -42,6 +41,10 @@ can also be imported as ``scrapy.linkextractors.LinkExtractor``:
 
     from scrapy.linkextractors import LinkExtractor
 
+.. autodata:: IGNORED_EXTENSIONS
+
+.. autodata:: SUPPORTED_SCHEMES
+
 LxmlLinkExtractor
 -----------------
 

scrapy/linkextractors/__init__.py

@@ -14,7 +14,6 @@ if TYPE_CHECKING:
     from collections.abc import Iterable
     from re import Pattern
 
-# common file extensions that are not followed if they occur in links
 IGNORED_EXTENSIONS = [
     # archives
     "7z",
@@ -114,20 +113,33 @@ IGNORED_EXTENSIONS = [
     "msp",
     "py",
 ]
+"""File extensions of files that are usually not worth following, and hence the
+default value of the ``deny_extensions`` parameter of :class:`LinkExtractor
+<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`."""
+
+
+SUPPORTED_SCHEMES = [
+    "data",
+    "file",
+    "ftp",
+    "http",
+    "https",
+    "s3",
+]
+"""URL schemes that Scrapy can download out of the box, and hence the default
+value of the ``schemes`` parameter of :class:`LinkExtractor
+<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`."""
 
 
 def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool:
     return any(r.search(url) for r in regexs)
 
 
-def _is_valid_url(url: str) -> bool:
-    return url.split("://", 1)[0] in {"http", "https", "file", "ftp"}
-
-
 # Top-level imports
 from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor
 
 __all__ = [
     "IGNORED_EXTENSIONS",
+    "SUPPORTED_SCHEMES",
     "LinkExtractor",
 ]

scrapy/linkextractors/lxmlhtml.py

@@ -18,7 +18,7 @@ from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
 
 from scrapy.link import Link
-from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches
+from scrapy.linkextractors import IGNORED_EXTENSIONS, SUPPORTED_SCHEMES, _matches
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list
 from scrapy.utils.response import get_base_url
@@ -202,6 +202,14 @@ class LxmlLinkExtractor:
         :data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
     :type deny_extensions: list
 
+    :param schemes: a single value or list of strings containing the URL
+        schemes that extracted links may use. If not given, it will default to
+        :data:`scrapy.linkextractors.SUPPORTED_SCHEMES`. Use ``()`` to extract
+        links regardless of their scheme.
+
+        .. versionadded:: VERSION
+    :type schemes: str or list
+
     :param restrict_xpaths: is an XPath (or list of XPath's) which defines
         regions inside the response where links should be extracted from.
         If given, only the text selected by those XPath will be scanned for
@@ -320,6 +328,7 @@ class LxmlLinkExtractor:
         restrict_text: _RegexOrSeveral | None = None,
         deny_tags: str | Iterable[str] = (),
         deny_attrs: str | Iterable[str] = (),
+        schemes: str | Iterable[str] | None = None,
     ):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         deny_tags, deny_attrs = (
@@ -347,8 +356,11 @@ class LxmlLinkExtractor:
 
         if deny_extensions is None:
             deny_extensions = IGNORED_EXTENSIONS
+        if schemes is None:
+            schemes = SUPPORTED_SCHEMES
         self.canonicalize: bool = canonicalize
         self.deny_extensions: set[str] = {"." + e for e in arg_to_iter(deny_extensions)}
+        self.schemes: set[str] = set(arg_to_iter(schemes))
         self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text)
 
     @staticmethod
@@ -359,13 +371,13 @@ class LxmlLinkExtractor:
         ]
 
     def _link_allowed(self, link: Link) -> bool:
-        if not _is_valid_url(link.url):
+        parsed_url = urlparse(link.url)
+        if self.schemes and parsed_url.scheme not in self.schemes:
             return False
         if self.allow_res and not _matches(link.url, self.allow_res):
             return False
         if self.deny_res and _matches(link.url, self.deny_res):
             return False
-        parsed_url = urlparse(link.url)
         if self.allow_domains and not url_is_from_any_domain(
             parsed_url, self.allow_domains
         ):

tests/test_linkextractors.py

@@ -691,6 +691,33 @@ class Base:
                 ),
             ]
 
+        def test_schemes(self):
+            body = b"""
+            <html><body>
+            <a href="page.html">Page</a>
+            <a href="data:text/plain,Data">Data</a>
+            <a href="mailto:someone@example.com">Mail</a>
+            </body></html>"""
+            response = HtmlResponse("http://www.example.com/index.html", body=body)
+
+            lx = self.extractor_cls()
+            assert lx.extract_links(response) == [
+                Link(url="http://www.example.com/page.html", text="Page"),
+                Link(url="data:text/plain,Data", text="Data"),
+            ]
+
+            lx = self.extractor_cls(schemes="mailto")
+            assert lx.extract_links(response) == [
+                Link(url="mailto:someone@example.com", text="Mail"),
+            ]
+
+            lx = self.extractor_cls(schemes=())
+            assert lx.extract_links(response) == [
+                Link(url="http://www.example.com/page.html", text="Page"),
+                Link(url="data:text/plain,Data", text="Data"),
+                Link(url="mailto:someone@example.com", text="Mail"),
+            ]
+
         def test_pickle_extractor(self):
             lx = self.extractor_cls()
             assert isinstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls)