diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 1c05ac20e..8f75e2618 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,34 +6,10 @@ library. from __future__ import annotations import re -import warnings -from importlib import import_module -from typing import TYPE_CHECKING, Any, TypeAlias +from typing import TYPE_CHECKING, TypeAlias from urllib.parse import ParseResult, urlparse, urlunparse -from w3lib.url import __all__ as _public_w3lib_objects -from w3lib.url import any_to_uri as _any_to_uri -from w3lib.url import parse_url as _parse_url - -from scrapy.exceptions import ScrapyDeprecationWarning - -_DEPRECATED_NAMES: frozenset[str] = frozenset( - {"_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects} -) - - -def __getattr__(name: str) -> Any: - if name in _DEPRECATED_NAMES: - obj_type = "attribute" if name == "_safe_chars" else "function" - warnings.warn( - f"The scrapy.utils.url.{name} {obj_type} is deprecated, use w3lib.url.{name} instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return getattr(import_module("w3lib.url"), name) - - raise AttributeError - +from w3lib.url import any_to_uri, parse_url if TYPE_CHECKING: from collections.abc import Iterable @@ -45,7 +21,7 @@ UrlT: TypeAlias = str | bytes | ParseResult def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: """Return True if the url belongs to any of the given domains""" - host = _parse_url(url).netloc.lower() + host = parse_url(url).netloc.lower() if not host: return False return any((host == d) or (host.endswith(f".{d}")) for d in map(str.lower, domains)) @@ -64,7 +40,7 @@ def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool: def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: """Return True if the url ends with one of the extensions provided""" - lowercase_path = _parse_url(url).path.lower() + lowercase_path = parse_url(url).path.lower() return any(lowercase_path.endswith(ext) for ext in extensions) @@ -125,7 +101,7 @@ def guess_scheme(url: str) -> str: """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" if _is_filesystem_path(url): - return _any_to_uri(url) + return any_to_uri(url) return add_http_if_no_scheme(url) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index bcbbe74a3..19a31c353 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,13 +1,9 @@ -import warnings -from importlib import import_module - import pytest from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.spiders import Spider -from scrapy.utils.url import ( # type: ignore[attr-defined] +from scrapy.utils.url import ( _is_filesystem_path, - _public_w3lib_objects, add_http_if_no_scheme, guess_scheme, strip_url, @@ -446,23 +442,3 @@ class TestStripUrl: ) def test__is_filesystem_path(path: str, expected: bool) -> None: assert _is_filesystem_path(path) == expected - - -@pytest.mark.parametrize( - "obj_name", - [ - "_unquotepath", - "_safe_chars", - "parse_url", - *_public_w3lib_objects, - ], -) -def test_deprecated_imports_from_w3lib(obj_name: str) -> None: - with warnings.catch_warnings(record=True) as warns: - obj_type = "attribute" if obj_name == "_safe_chars" else "function" - message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead." - - getattr(import_module("scrapy.utils.url"), obj_name) - - assert isinstance(warns[0].message, Warning) - assert message in warns[0].message.args