Deprecate w3lib objects importable from scrapy.utils.url (#6586)

scrapy/utils/url.py

@@ -1,36 +1,47 @@
 """
 This module contains general purpose URL functions not found in the standard
 library.
-
-Some of the functions that used to be imported from this module have been moved
-to the w3lib.url module. Always import those from there instead.
 """
 
 from __future__ import annotations
 
 import re
-from typing import TYPE_CHECKING, Union, cast
+import warnings
+from importlib import import_module
+from typing import TYPE_CHECKING, Union
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 
-# scrapy.utils.url was moved to w3lib.url and import * ensures this
-# move doesn't break old code
-from w3lib.url import *  # pylint: disable=unused-wildcard-import,wildcard-import
-from w3lib.url import _safe_chars, _unquotepath  # noqa: F401
+from w3lib.url import __all__ as _public_w3lib_objects
+from w3lib.url import add_or_replace_parameter as _add_or_replace_parameter
+from w3lib.url import any_to_uri as _any_to_uri
+from w3lib.url import parse_url as _parse_url
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+
+def __getattr__(name: str):
+    if name in ("_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects):
+        obj_type = "attribute" if name == "_safe_chars" else "function"
+        warnings.warn(
+            f"The scrapy.utils.url.{name} {obj_type} is deprecated, use w3lib.url.{name} instead.",
+            ScrapyDeprecationWarning,
+        )
+        return getattr(import_module("w3lib.url"), name)
+
+    raise AttributeError
 
-from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
     from scrapy import Spider
 
-
 UrlT = Union[str, bytes, ParseResult]
 
 
 def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:
     """Return True if the url belongs to any of the given domains"""
-    host = parse_url(url).netloc.lower()
+    host = _parse_url(url).netloc.lower()
     if not host:
         return False
     domains = [d.lower() for d in domains]
@@ -46,21 +57,10 @@ def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool:
 
 def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:
     """Return True if the url ends with one of the extensions provided"""
-    lowercase_path = parse_url(url).path.lower()
+    lowercase_path = _parse_url(url).path.lower()
     return any(lowercase_path.endswith(ext) for ext in extensions)
 
 
-def parse_url(  # pylint: disable=function-redefined
-    url: UrlT, encoding: str | None = None
-) -> ParseResult:
-    """Return urlparsed url from the given argument (which could be an already
-    parsed url)
-    """
-    if isinstance(url, ParseResult):
-        return url
-    return cast(ParseResult, urlparse(to_unicode(url, encoding)))
-
-
 def escape_ajax(url: str) -> str:
     """
     Return the crawlable url
@@ -86,7 +86,7 @@ def escape_ajax(url: str) -> str:
     defrag, frag = urldefrag(url)
     if not frag.startswith("!"):
         return url
-    return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:])
+    return _add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:])
 
 
 def add_http_if_no_scheme(url: str) -> str:
@@ -146,7 +146,7 @@ def guess_scheme(url: str) -> str:
     """Add an URL scheme if missing: file:// for filepath-like input or
     http:// otherwise."""
     if _is_filesystem_path(url):
-        return any_to_uri(url)
+        return _any_to_uri(url)
     return add_http_if_no_scheme(url)
 
 

tests/test_utils_url.py

@@ -1,10 +1,14 @@
 import unittest
+import warnings
+
+import pytest
 
 from scrapy.linkextractors import IGNORED_EXTENSIONS
 from scrapy.spiders import Spider
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.url import (
     _is_filesystem_path,
+    _public_w3lib_objects,
     add_http_if_no_scheme,
     guess_scheme,
     strip_url,
@@ -607,5 +611,26 @@ class IsPathTestCase(unittest.TestCase):
             )
 
 
+@pytest.mark.parametrize(
+    "obj_name",
+    [
+        "_unquotepath",
+        "_safe_chars",
+        "parse_url",
+        *_public_w3lib_objects,
+    ],
+)
+def test_deprecated_imports_from_w3lib(obj_name):
+    with warnings.catch_warnings(record=True) as warns:
+        obj_type = "attribute" if obj_name == "_safe_chars" else "function"
+        message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."
+
+        from importlib import import_module
+
+        getattr(import_module("scrapy.utils.url"), obj_name)
+
+        assert message in warns[0].message.args
+
+
 if __name__ == "__main__":
     unittest.main()