diff --git a/pyproject.toml b/pyproject.toml index 0bdcf6b51..85dde855b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "cryptography>=37.0.0", "cssselect>=0.9.1", "defusedxml>=0.7.1", + "formerly>=0.1.0", "itemadapter>=0.1.0", "itemloaders>=1.0.1", "lxml>=4.6.4", @@ -21,6 +22,7 @@ dependencies = [ "queuelib>=1.6.1", "service_identity>=23.1.0", "tldextract", + 'typing-extensions>=4.5.0; python_version < "3.13"', "w3lib>=1.17.0", "zope.interface>=5.1.0", # Platform-specific dependencies @@ -248,10 +250,10 @@ disable = [ "disallowed-name", "duplicate-code", # https://github.com/pylint-dev/pylint/issues/214 "fixme", - "inherit-non-class", # false positives with create_deprecated_class() + "inherit-non-class", # false positives with formerly.deprecated_class() "invalid-name", "invalid-overridden-method", - "isinstance-second-argument-not-valid-type", # false positives with create_deprecated_class() + "isinstance-second-argument-not-valid-type", # false positives with formerly.deprecated_class() "line-too-long", "logging-format-interpolation", "logging-fstring-interpolation", diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index ef948997d..c7ea9a326 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -3,6 +3,7 @@ from __future__ import annotations import warnings from typing import TYPE_CHECKING, Any, cast +from formerly import deprecated_class from OpenSSL import SSL from twisted.internet.ssl import ( AcceptableCiphers, @@ -22,7 +23,6 @@ from scrapy.core.downloader.tls import ( ) from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils._deps_compat import TWISTED_TLS_NEW_IMPL -from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.ssl import _get_cert_options_version_kwargs, _get_tls_version_limits @@ -160,11 +160,12 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): ) -ScrapyClientContextFactory = create_deprecated_class( +ScrapyClientContextFactory = deprecated_class( "ScrapyClientContextFactory", _ScrapyClientContextFactory, - subclass_warn_message="{old} is deprecated.", - instance_warn_message="{cls} is deprecated.", + category=ScrapyDeprecationWarning, + subclass_message="{old} is deprecated.", + instance_message="{cls} is deprecated.", ) @@ -247,11 +248,12 @@ class _AcceptableProtocolsContextFactory: return options -AcceptableProtocolsContextFactory = create_deprecated_class( +AcceptableProtocolsContextFactory = deprecated_class( "AcceptableProtocolsContextFactory", _AcceptableProtocolsContextFactory, - subclass_warn_message="{old} is deprecated.", - instance_warn_message="{cls} is deprecated.", + category=ScrapyDeprecationWarning, + subclass_message="{old} is deprecated.", + instance_message="{cls} is deprecated.", ) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index c5cf1156d..d7001c971 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -4,6 +4,7 @@ import logging import warnings from typing import TYPE_CHECKING, Any +from formerly import deprecated_class from OpenSSL import SSL from service_identity import VerificationError from service_identity.exceptions import CertificateError @@ -22,7 +23,6 @@ from twisted.internet._sslverify import ClientTLSOptions from twisted.internet.ssl import AcceptableCiphers, TLSVersion from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.deprecate import create_deprecated_class if TYPE_CHECKING: from collections.abc import Callable @@ -117,11 +117,12 @@ class _ScrapyClientTLSOptions(ClientTLSOptions): super()._identityVerifyingInfoCallback(connection, where, ret) # type: ignore[misc] -ScrapyClientTLSOptions = create_deprecated_class( +ScrapyClientTLSOptions = deprecated_class( "ScrapyClientTLSOptions", _ScrapyClientTLSOptions, - subclass_warn_message="{old} is deprecated.", - instance_warn_message="{cls} is deprecated.", + category=ScrapyDeprecationWarning, + subclass_message="{old} is deprecated.", + instance_message="{cls} is deprecated.", ) diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index bd9c435de..f98a0ee55 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -1,6 +1,7 @@ from __future__ import annotations import inspect +import sys import warnings from functools import wraps from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast, overload @@ -12,6 +13,11 @@ from scrapy.utils.asyncio import run_in_thread from scrapy.utils.defer import deferred_from_coro from scrapy.utils.python import _signature +if sys.version_info >= (3, 13): + from warnings import deprecated as _deprecated +else: + from typing_extensions import deprecated as _deprecated + if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Coroutine @@ -30,12 +36,15 @@ def deprecated( ) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ... +@_deprecated( + "scrapy.utils.decorators.deprecated() is deprecated, use warnings.deprecated()" + " instead.", + category=ScrapyDeprecationWarning, +) def deprecated( use_instead: Callable[_P, _T] | str | None = None, ) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]: - """This is a decorator which can be used to mark functions - as deprecated. It will result in a warning being emitted - when the function is used.""" + """Mark a function as deprecated, so that calling it warns.""" def deco(func: Callable[_P, _T]) -> Callable[_P, _T]: @wraps(func) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 4fd50fdad..a53802291 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -3,16 +3,27 @@ from __future__ import annotations import inspect +import sys import warnings -from typing import TYPE_CHECKING, Any, cast, overload +from typing import TYPE_CHECKING, Any, overload + +from formerly import deprecated_class from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import get_func_args_dict +if sys.version_info >= (3, 13): + from warnings import deprecated as _deprecated +else: + from typing_extensions import deprecated as _deprecated + if TYPE_CHECKING: from collections.abc import Callable +_WRAPPER_MODULES = frozenset({__name__, _deprecated.__module__}) + + def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> None: cname = obj.__class__.__name__ warnings.warn( @@ -23,6 +34,11 @@ def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> No ) +@_deprecated( + "scrapy.utils.deprecate.create_deprecated_class() is deprecated, use" + " formerly.deprecated_class() instead.", + category=ScrapyDeprecationWarning, +) def create_deprecated_class( name: str, new_class: type, @@ -34,114 +50,27 @@ def create_deprecated_class( subclass_warn_message: str = "{cls} inherits from deprecated class {old}, please inherit from {new}.", instance_warn_message: str = "{cls} is deprecated, instantiate {new} instead.", ) -> type: - """ - Return a "deprecated" class that causes its subclasses to issue a warning. - Subclasses of ``new_class`` are considered subclasses of this class. - It also warns when the deprecated class is instantiated, but do not when - its subclasses are instantiated. - - It can be used to rename a base class in a library. For example, if we - have - - .. code-block:: python - - class OldName(SomeClass): ... - - and we want to rename it to NewName, we can do the following: - - .. code-block:: python - - class NewName(SomeClass): ... - - - OldName = create_deprecated_class("OldName", NewName) - - Then, if user class inherits from OldName, warning is issued. Also, if - some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)`` - checks they'll still return True if sub is a subclass of NewName instead of - OldName. - """ - - # https://github.com/python/mypy/issues/4177 - class DeprecatedClass(new_class.__class__): # type: ignore[misc,name-defined] - # pylint: disable=no-self-argument - deprecated_class: type | None = None - warned_on_subclass: bool = False - - def __new__( # pylint: disable=bad-classmethod-argument - metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any] - ) -> type: - cls: type = super().__new__(metacls, name, bases, clsdict_) - if metacls.deprecated_class is None: - metacls.deprecated_class = cls - return cls - - def __init__(cls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]): - meta = cls.__class__ - old = meta.deprecated_class - if old in bases and not (warn_once and meta.warned_on_subclass): - meta.warned_on_subclass = True - msg = subclass_warn_message.format( - cls=_clspath(cls), - old=_clspath(old, old_class_path), - new=_clspath(new_class, new_class_path), - ) - if warn_once: - msg += " (warning only on first subclass, there may be others)" - warnings.warn(msg, warn_category, stacklevel=2) - super().__init__(name, bases, clsdict_) - - # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass - # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks - # for implementation details - def __instancecheck__(cls, inst: Any) -> bool: - return any(cls.__subclasscheck__(c) for c in (type(inst), inst.__class__)) - - def __subclasscheck__(cls, sub: type) -> bool: - if cls is not DeprecatedClass.deprecated_class: - # we should do the magic only if second `issubclass` argument - # is the deprecated class itself - subclasses of the - # deprecated class should not use custom `__subclasscheck__` - # method. - return cast("bool", super().__subclasscheck__(sub)) - - if not inspect.isclass(sub): - raise TypeError("issubclass() arg 1 must be a class") - - mro = getattr(sub, "__mro__", ()) - return any(c in {cls, new_class} for c in mro) - - def __call__(cls, *args: Any, **kwargs: Any) -> Any: - old = DeprecatedClass.deprecated_class - if cls is old: - msg = instance_warn_message.format( - cls=_clspath(cls, old_class_path), - new=_clspath(new_class, new_class_path), - ) - warnings.warn(msg, warn_category, stacklevel=2) - return super().__call__(*args, **kwargs) - - deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {}) - - try: - frm = inspect.stack()[1] - parent_module = inspect.getmodule(frm[0]) - if parent_module is not None: - deprecated_cls.__module__ = parent_module.__name__ - except Exception as e: - # Sometimes inspect.stack() fails (e.g. when the first import of - # deprecated class is in jinja2 template). __module__ attribute is not - # important enough to raise an exception as users may be unable - # to fix inspect.stack() errors. - warnings.warn(f"Error detecting parent module: {e!r}", stacklevel=2) - - return deprecated_cls - - -def _clspath(cls: type, forced: str | None = None) -> str: - if forced is not None: - return forced - return f"{cls.__module__}.{cls.__name__}" + """Return a deprecated alias of *new_class* named *name*.""" + cls: type = deprecated_class( + name, + new_class, + namespace=clsdict, + category=warn_category, + warn_once=warn_once, + old_path=old_class_path, + new_path=new_class_path, + subclass_message=subclass_warn_message, + instance_message=instance_warn_message, + ) + # deprecated_class() takes the module of the alias from its calling frame, + # which is this function and the decorator wrapping it, so skip past both. + frame = inspect.currentframe() + assert frame is not None + while frame.f_globals.get("__name__") in _WRAPPER_MODULES: + assert frame.f_back is not None + frame = frame.f_back + cls.__module__ = frame.f_globals.get("__name__", cls.__module__) + return cls DEPRECATION_RULES: list[tuple[str, str]] = [] diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py index 92677882e..d4e9e8129 100644 --- a/tests/test_utils_decorators.py +++ b/tests/test_utils_decorators.py @@ -18,7 +18,12 @@ if TYPE_CHECKING: class TestDeprecated: def test_warns_and_still_calls(self): - @deprecated() + with pytest.warns( + ScrapyDeprecationWarning, match=r"decorators\.deprecated\(\) is deprecated" + ): + decorate = deprecated() + + @decorate def add(a: int, b: int) -> int: return a + b @@ -30,7 +35,11 @@ class TestDeprecated: assert result == 5 def test_use_instead_in_message(self): - @deprecated(use_instead="other_function") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + decorate = deprecated(use_instead="other_function") + + @decorate def old() -> None: return None @@ -41,10 +50,13 @@ class TestDeprecated: old() def test_applied_without_parentheses(self): - @deprecated def square(x: int) -> int: return x * x + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + square = deprecated(square) + with pytest.warns( ScrapyDeprecationWarning, match=r"Call to deprecated function square\." ) as record: diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 4c8585916..5d95e4776 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,256 +1,55 @@ from __future__ import annotations -import inspect import warnings from unittest import mock import pytest from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.deprecate import create_deprecated_class, update_classpath +from scrapy.utils.deprecate import attribute, create_deprecated_class, update_classpath -class MyWarning(UserWarning): +class NewName: pass -class SomeBaseClass: - pass +def test_attribute(): + with pytest.warns( + ScrapyDeprecationWarning, + match=r"NewName\.old attribute is deprecated and will be no longer supported" + r" in Scrapy 1\.0, use NewName\.new attribute instead", + ): + attribute(NewName(), "old", "new", version="1.0") -class NewName(SomeBaseClass): - pass - - -class TestWarnWhenSubclassed: - def test_no_warning_on_definition(self): - with warnings.catch_warnings(): - warnings.simplefilter("error", category=ScrapyDeprecationWarning) +class TestCreateDeprecatedClass: + def test_warns_about_itself(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"create_deprecated_class\(\) is deprecated" + ): create_deprecated_class("Deprecated", NewName) - def test_subclassing_warning_message(self): - msg = ( - r"tests\.test_utils_deprecate\.UserClass inherits from " - r"deprecated class tests\.test_utils_deprecate\.Deprecated, " - r"please inherit from tests\.test_utils_deprecate\.NewName." - r" \(warning only on first subclass, there may be others\)" - ) - Deprecated = create_deprecated_class( - "Deprecated", NewName, warn_category=MyWarning - ) - with pytest.warns(MyWarning, match=msg) as w: - - class UserClass(Deprecated): # type: ignore[misc, valid-type] - pass - - assert w[0].lineno == inspect.getsourcelines(UserClass)[1] - - def test_custom_class_paths(self): - Deprecated = create_deprecated_class( - "Deprecated", - NewName, - new_class_path="foo.NewClass", - old_class_path="bar.OldClass", - warn_category=MyWarning, - ) - - with pytest.warns( - MyWarning, - match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass", - ): - - class UserClass(Deprecated): # type: ignore[misc, valid-type] - pass - - with pytest.warns( - MyWarning, - match=r"bar\.OldClass is deprecated, instantiate foo\.NewClass instead", - ): - _ = Deprecated() - - def test_subclassing_warns_only_on_direct_children(self): - Deprecated = create_deprecated_class( - "Deprecated", NewName, warn_once=False, warn_category=MyWarning - ) - - with pytest.warns( - MyWarning, - match="UserClass inherits from deprecated class", - ): - - class UserClass(Deprecated): # type: ignore[misc, valid-type] - pass - - with warnings.catch_warnings(): - warnings.simplefilter("error", MyWarning) - - class NoWarnOnMe(UserClass): - pass - - def test_subclassing_warns_once_by_default(self): - Deprecated = create_deprecated_class( - "Deprecated", NewName, warn_category=MyWarning - ) - - with pytest.warns( - MyWarning, - match="UserClass inherits from deprecated class", - ): - - class UserClass(Deprecated): # type: ignore[misc, valid-type] - pass - - with warnings.catch_warnings(): - warnings.simplefilter("error", MyWarning) - - class FooClass(Deprecated): # type: ignore[misc, valid-type] - pass - - class BarClass(Deprecated): # type: ignore[misc, valid-type] - pass - - def test_warning_on_instance(self): - Deprecated = create_deprecated_class( - "Deprecated", NewName, warn_category=MyWarning - ) - - with pytest.warns( - MyWarning, - match=r"tests\.test_utils_deprecate\.Deprecated is deprecated, " - r"instantiate tests\.test_utils_deprecate\.NewName instead\.", - ) as w: - _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) # type: ignore[arg-type] - assert len(w) == 1 - assert w[0].lineno == lineno - - # ignore subclassing warnings - with warnings.catch_warnings(): - warnings.simplefilter("ignore", MyWarning) - - class UserClass(Deprecated): # type: ignore[misc, valid-type] - pass - - with warnings.catch_warnings(): - warnings.simplefilter("error", MyWarning) - UserClass() # subclass instances don't warn - - def test_warning_auto_message(self): - Deprecated = create_deprecated_class("Deprecated", NewName) - with pytest.warns( - ScrapyDeprecationWarning, - match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName", - ): - - class UserClass2(Deprecated): # type: ignore[misc, valid-type] - pass - - def test_issubclass(self): + def test_returns_a_working_alias(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) - DeprecatedName = create_deprecated_class("DeprecatedName", NewName) - - class UpdatedUserClass1(NewName): - pass - - class UpdatedUserClass1a(NewName): - pass - - class OutdatedUserClass1(DeprecatedName): # type: ignore[misc, valid-type] - pass - - class OutdatedUserClass1a(DeprecatedName): # type: ignore[misc, valid-type] - pass - - class UnrelatedClass: - pass - - assert issubclass(UpdatedUserClass1, NewName) - assert issubclass(UpdatedUserClass1a, NewName) - assert issubclass(UpdatedUserClass1, DeprecatedName) - assert issubclass(UpdatedUserClass1a, DeprecatedName) - assert issubclass(OutdatedUserClass1, DeprecatedName) - assert not issubclass(UnrelatedClass, DeprecatedName) - assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a) - assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1) - - with pytest.raises(TypeError): - issubclass(object(), DeprecatedName) # type: ignore[arg-type] - - def test_isinstance(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - DeprecatedName = create_deprecated_class("DeprecatedName", NewName) - - class UpdatedUserClass2(NewName): - pass - - class UpdatedUserClass2a(NewName): - pass - - class OutdatedUserClass2(DeprecatedName): # type: ignore[misc, valid-type] - pass - - class OutdatedUserClass2a(DeprecatedName): # type: ignore[misc, valid-type] - pass - - class UnrelatedClass: - pass - - assert isinstance(UpdatedUserClass2(), NewName) - assert isinstance(UpdatedUserClass2a(), NewName) - assert isinstance(UpdatedUserClass2(), DeprecatedName) - assert isinstance(UpdatedUserClass2a(), DeprecatedName) - assert isinstance(OutdatedUserClass2(), DeprecatedName) - assert isinstance(OutdatedUserClass2a(), DeprecatedName) - assert not isinstance(OutdatedUserClass2a(), OutdatedUserClass2) - assert not isinstance(OutdatedUserClass2(), OutdatedUserClass2a) - assert not isinstance(UnrelatedClass(), DeprecatedName) - - def test_clsdict(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"}) + Deprecated = create_deprecated_class( + "Deprecated", NewName, {"foo": "bar"}, warn_once=False + ) + assert Deprecated.__module__ == __name__ assert Deprecated.foo == "bar" # type: ignore[attr-defined] - def test_deprecate_a_class_with_custom_metaclass(self): - Meta1 = type("Meta1", (type,), {}) - New = Meta1("New", (), {}) - create_deprecated_class("Deprecated", New) - - def test_deprecate_subclass_of_deprecated_class(self): - with warnings.catch_warnings(): - warnings.simplefilter("error", MyWarning) - Deprecated = create_deprecated_class( - "Deprecated", NewName, warn_category=MyWarning - ) - AlsoDeprecated = create_deprecated_class( - "AlsoDeprecated", - Deprecated, - new_class_path="foo.Bar", - warn_category=MyWarning, - ) - with pytest.warns( - MyWarning, - match=r"AlsoDeprecated is deprecated, instantiate foo\.Bar instead", - ): - AlsoDeprecated() - - with pytest.warns( - MyWarning, - match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar", + ScrapyDeprecationWarning, + match=r"tests\.test_utils_deprecate\.UserClass inherits from deprecated" + r" class tests\.test_utils_deprecate\.Deprecated, please inherit from" + r" tests\.test_utils_deprecate\.NewName\.", ): - class UserClass(AlsoDeprecated): # type: ignore[misc, valid-type] + class UserClass(Deprecated): # type: ignore[misc, valid-type] pass - def test_inspect_stack(self): - with ( - mock.patch("inspect.stack", side_effect=IndexError), - pytest.warns(UserWarning, match="Error detecting parent module"), - ): - create_deprecated_class("DeprecatedName", NewName) + assert issubclass(UserClass, Deprecated) @mock.patch( diff --git a/tox.ini b/tox.ini index 7ea1c57d7..7d34426f0 100644 --- a/tox.ini +++ b/tox.ini @@ -142,6 +142,7 @@ deps = brotlicffi==1.2.0.0; implementation_name == "pypy" cryptography==37.0.0 cssselect==0.9.1 + formerly==0.1.0 httpx2==2.0.0 itemadapter==0.1.0 lxml==4.6.4 @@ -149,6 +150,7 @@ deps = pyOpenSSL==22.0.0 queuelib==1.6.1 service_identity==23.1.0 + typing-extensions==4.5.0 w3lib==1.17.0 zope.interface==5.1.0 {[test-requirements]deps}