diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 833aa3e20..22b4197f9 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,6 +6,7 @@ Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ import re +from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this @@ -15,8 +16,14 @@ from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + from scrapy import Spider -def url_is_from_any_domain(url, domains): + +UrlT = Union[str, bytes, ParseResult] + + +def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: """Return True if the url belongs to any of the given domains""" host = parse_url(url).netloc.lower() if not host: @@ -25,29 +32,29 @@ def url_is_from_any_domain(url, domains): return any((host == d) or (host.endswith(f".{d}")) for d in domains) -def url_is_from_spider(url, spider): +def url_is_from_spider(url: UrlT, spider: Type["Spider"]) -> bool: """Return True if the url belongs to the given spider""" return url_is_from_any_domain( url, [spider.name] + list(getattr(spider, "allowed_domains", [])) ) -def url_has_any_extension(url, extensions): +def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: """Return True if the url ends with one of the extensions provided""" lowercase_path = parse_url(url).path.lower() return any(lowercase_path.endswith(ext) for ext in extensions) -def parse_url(url, encoding=None): +def parse_url(url: UrlT, encoding: Optional[str] = None) -> ParseResult: """Return urlparsed url from the given argument (which could be an already parsed url) """ if isinstance(url, ParseResult): return url - return urlparse(to_unicode(url, encoding)) + return cast(ParseResult, urlparse(to_unicode(url, encoding))) -def escape_ajax(url): +def escape_ajax(url: str) -> str: """ Return the crawlable url according to: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started @@ -76,7 +83,7 @@ def escape_ajax(url): return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:]) -def add_http_if_no_scheme(url): +def add_http_if_no_scheme(url: str) -> str: """Add http as the default scheme if it is missing from the url.""" match = re.match(r"^\w+://", url, flags=re.I) if not match: @@ -87,7 +94,7 @@ def add_http_if_no_scheme(url): return url -def _is_posix_path(string): +def _is_posix_path(string: str) -> bool: return bool( re.match( r""" @@ -109,7 +116,7 @@ def _is_posix_path(string): ) -def _is_windows_path(string): +def _is_windows_path(string: str) -> bool: return bool( re.match( r""" @@ -125,11 +132,11 @@ def _is_windows_path(string): ) -def _is_filesystem_path(string): +def _is_filesystem_path(string: str) -> bool: return _is_posix_path(string) or _is_windows_path(string) -def guess_scheme(url): +def guess_scheme(url: str) -> str: """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" if _is_filesystem_path(url): @@ -138,12 +145,12 @@ def guess_scheme(url): def strip_url( - url, - strip_credentials=True, - strip_default_port=True, - origin_only=False, - strip_fragment=True, -): + url: str, + strip_credentials: bool = True, + strip_default_port: bool = True, + origin_only: bool = False, + strip_fragment: bool = True, +) -> str: """Strip URL string from some of its components: - ``strip_credentials`` removes "user:password@"