diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index c1d872dd0..ff40f52b4 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -32,7 +32,6 @@ Link extractor reference ======================== .. module:: scrapy.linkextractors - :synopsis: Link extractors classes The link extractor class is :class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it @@ -42,6 +41,10 @@ can also be imported as ``scrapy.linkextractors.LinkExtractor``: from scrapy.linkextractors import LinkExtractor +.. autodata:: IGNORED_EXTENSIONS + +.. autodata:: SUPPORTED_SCHEMES + LxmlLinkExtractor ----------------- diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index b39859f7b..cb4d22022 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -14,7 +14,6 @@ if TYPE_CHECKING: from collections.abc import Iterable from re import Pattern -# common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ # archives "7z", @@ -114,20 +113,33 @@ IGNORED_EXTENSIONS = [ "msp", "py", ] +"""File extensions of files that are usually not worth following, and hence the +default value of the ``deny_extensions`` parameter of :class:`LinkExtractor +`.""" + + +SUPPORTED_SCHEMES = [ + "data", + "file", + "ftp", + "http", + "https", + "s3", +] +"""URL schemes that Scrapy can download out of the box, and hence the default +value of the ``schemes`` parameter of :class:`LinkExtractor +`.""" def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool: return any(r.search(url) for r in regexs) -def _is_valid_url(url: str) -> bool: - return url.split("://", 1)[0] in {"http", "https", "file", "ftp"} - - # Top-level imports from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor __all__ = [ "IGNORED_EXTENSIONS", + "SUPPORTED_SCHEMES", "LinkExtractor", ] diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 75f9753e7..aeb7def94 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -18,7 +18,7 @@ from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link -from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches +from scrapy.linkextractors import IGNORED_EXTENSIONS, SUPPORTED_SCHEMES, _matches from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url @@ -202,6 +202,14 @@ class LxmlLinkExtractor: :data:`scrapy.linkextractors.IGNORED_EXTENSIONS`. :type deny_extensions: list + :param schemes: a single value or list of strings containing the URL + schemes that extracted links may use. If not given, it will default to + :data:`scrapy.linkextractors.SUPPORTED_SCHEMES`. Use ``()`` to extract + links regardless of their scheme. + + .. versionadded:: VERSION + :type schemes: str or list + :param restrict_xpaths: is an XPath (or list of XPath's) which defines regions inside the response where links should be extracted from. If given, only the text selected by those XPath will be scanned for @@ -320,6 +328,7 @@ class LxmlLinkExtractor: restrict_text: _RegexOrSeveral | None = None, deny_tags: str | Iterable[str] = (), deny_attrs: str | Iterable[str] = (), + schemes: str | Iterable[str] | None = None, ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) deny_tags, deny_attrs = ( @@ -347,8 +356,11 @@ class LxmlLinkExtractor: if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS + if schemes is None: + schemes = SUPPORTED_SCHEMES self.canonicalize: bool = canonicalize self.deny_extensions: set[str] = {"." + e for e in arg_to_iter(deny_extensions)} + self.schemes: set[str] = set(arg_to_iter(schemes)) self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text) @staticmethod @@ -359,13 +371,13 @@ class LxmlLinkExtractor: ] def _link_allowed(self, link: Link) -> bool: - if not _is_valid_url(link.url): + parsed_url = urlparse(link.url) + if self.schemes and parsed_url.scheme not in self.schemes: return False if self.allow_res and not _matches(link.url, self.allow_res): return False if self.deny_res and _matches(link.url, self.deny_res): return False - parsed_url = urlparse(link.url) if self.allow_domains and not url_is_from_any_domain( parsed_url, self.allow_domains ): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 7b73a133f..80bbed8b5 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -691,6 +691,33 @@ class Base: ), ] + def test_schemes(self): + body = b""" + + Page + Data + Mail + """ + response = HtmlResponse("http://www.example.com/index.html", body=body) + + lx = self.extractor_cls() + assert lx.extract_links(response) == [ + Link(url="http://www.example.com/page.html", text="Page"), + Link(url="data:text/plain,Data", text="Data"), + ] + + lx = self.extractor_cls(schemes="mailto") + assert lx.extract_links(response) == [ + Link(url="mailto:someone@example.com", text="Mail"), + ] + + lx = self.extractor_cls(schemes=()) + assert lx.extract_links(response) == [ + Link(url="http://www.example.com/page.html", text="Page"), + Link(url="data:text/plain,Data", text="Data"), + Link(url="mailto:someone@example.com", text="Mail"), + ] + def test_pickle_extractor(self): lx = self.extractor_cls() assert isinstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls)