RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge cf1bb71be0 into ad43bf0c56

This commit is contained in:
Adrian 2026-08-15 11:31:51 -05:00 committed by GitHub
parent ad43bf0c56 cf1bb71be0
commit db606807e5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 110 additions and 354 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pyproject.toml
View File

@ -11,6 +11,7 @@ dependencies = [
"cryptography>=37.0.0",
"cssselect>=0.9.1",
"defusedxml>=0.7.1",
"formerly>=0.1.0",
"itemadapter>=0.1.0",
"itemloaders>=1.0.1",
"lxml>=4.6.4",
@ -21,6 +22,7 @@ dependencies = [
"queuelib>=1.6.1",
"service_identity>=23.1.0",
"tldextract",
'typing-extensions>=4.5.0; python_version < "3.13"',
"w3lib>=1.17.0",
"zope.interface>=5.1.0",
# Platform-specific dependencies
@ -248,10 +250,10 @@ disable = [
"disallowed-name",
"duplicate-code", # https://github.com/pylint-dev/pylint/issues/214
"fixme",
"inherit-non-class", # false positives with create_deprecated_class()
"inherit-non-class", # false positives with formerly.deprecated_class()
"invalid-name",
"invalid-overridden-method",
"isinstance-second-argument-not-valid-type", # false positives with create_deprecated_class()
"isinstance-second-argument-not-valid-type", # false positives with formerly.deprecated_class()
"line-too-long",
"logging-format-interpolation",
"logging-fstring-interpolation",

16
scrapy/core/downloader/contextfactory.py
View File

@ -3,6 +3,7 @@ from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Any, cast
from formerly import deprecated_class
from OpenSSL import SSL
from twisted.internet.ssl import (
AcceptableCiphers,
@ -22,7 +23,6 @@ from scrapy.core.downloader.tls import (
)
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils._deps_compat import TWISTED_TLS_NEW_IMPL
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.ssl import _get_cert_options_version_kwargs, _get_tls_version_limits
@ -160,11 +160,12 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
)
ScrapyClientContextFactory = create_deprecated_class(
ScrapyClientContextFactory = deprecated_class(
"ScrapyClientContextFactory",
_ScrapyClientContextFactory,
subclass_warn_message="{old} is deprecated.",
instance_warn_message="{cls} is deprecated.",
category=ScrapyDeprecationWarning,
subclass_message="{old} is deprecated.",
instance_message="{cls} is deprecated.",
)
@ -247,11 +248,12 @@ class _AcceptableProtocolsContextFactory:
return options
AcceptableProtocolsContextFactory = create_deprecated_class(
AcceptableProtocolsContextFactory = deprecated_class(
"AcceptableProtocolsContextFactory",
_AcceptableProtocolsContextFactory,
subclass_warn_message="{old} is deprecated.",
instance_warn_message="{cls} is deprecated.",
category=ScrapyDeprecationWarning,
subclass_message="{old} is deprecated.",
instance_message="{cls} is deprecated.",
)

9
scrapy/core/downloader/tls.py
View File

@ -4,6 +4,7 @@ import logging
import warnings
from typing import TYPE_CHECKING, Any
from formerly import deprecated_class
from OpenSSL import SSL
from service_identity import VerificationError
from service_identity.exceptions import CertificateError
@ -22,7 +23,6 @@ from twisted.internet._sslverify import ClientTLSOptions
from twisted.internet.ssl import AcceptableCiphers, TLSVersion
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import create_deprecated_class
if TYPE_CHECKING:
from collections.abc import Callable
@ -117,11 +117,12 @@ class _ScrapyClientTLSOptions(ClientTLSOptions):
super()._identityVerifyingInfoCallback(connection, where, ret) # type: ignore[misc]
ScrapyClientTLSOptions = create_deprecated_class(
ScrapyClientTLSOptions = deprecated_class(
"ScrapyClientTLSOptions",
_ScrapyClientTLSOptions,
subclass_warn_message="{old} is deprecated.",
instance_warn_message="{cls} is deprecated.",
category=ScrapyDeprecationWarning,
subclass_message="{old} is deprecated.",
instance_message="{cls} is deprecated.",
)

15
scrapy/utils/decorators.py
View File

@ -1,6 +1,7 @@
from __future__ import annotations
import inspect
import sys
import warnings
from functools import wraps
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast, overload
@ -12,6 +13,11 @@ from scrapy.utils.asyncio import run_in_thread
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.python import _signature
if sys.version_info >= (3, 13):
from warnings import deprecated as _deprecated
else:
from typing_extensions import deprecated as _deprecated
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, Coroutine
@ -30,12 +36,15 @@ def deprecated(
) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ...
@_deprecated(
"scrapy.utils.decorators.deprecated() is deprecated, use warnings.deprecated()"
" instead.",
category=ScrapyDeprecationWarning,
)
def deprecated(
use_instead: Callable[_P, _T] | str | None = None,
) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]:
"""This is a decorator which can be used to mark functions
as deprecated. It will result in a warning being emitted
when the function is used."""
"""Mark a function as deprecated, so that calling it warns."""
def deco(func: Callable[_P, _T]) -> Callable[_P, _T]:
@wraps(func)

147
scrapy/utils/deprecate.py
View File

@ -3,16 +3,27 @@
from __future__ import annotations
import inspect
import sys
import warnings
from typing import TYPE_CHECKING, Any, cast, overload
from typing import TYPE_CHECKING, Any, overload
from formerly import deprecated_class
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.python import get_func_args_dict
if sys.version_info >= (3, 13):
from warnings import deprecated as _deprecated
else:
from typing_extensions import deprecated as _deprecated
if TYPE_CHECKING:
from collections.abc import Callable
_WRAPPER_MODULES = frozenset({__name__, _deprecated.__module__})
def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> None:
cname = obj.__class__.__name__
warnings.warn(
@ -23,6 +34,11 @@ def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> No
)
@_deprecated(
"scrapy.utils.deprecate.create_deprecated_class() is deprecated, use"
" formerly.deprecated_class() instead.",
category=ScrapyDeprecationWarning,
)
def create_deprecated_class(
name: str,
new_class: type,
@ -34,114 +50,27 @@ def create_deprecated_class(
subclass_warn_message: str = "{cls} inherits from deprecated class {old}, please inherit from {new}.",
instance_warn_message: str = "{cls} is deprecated, instantiate {new} instead.",
) -> type:
"""
Return a "deprecated" class that causes its subclasses to issue a warning.
Subclasses of ``new_class`` are considered subclasses of this class.
It also warns when the deprecated class is instantiated, but do not when
its subclasses are instantiated.
It can be used to rename a base class in a library. For example, if we
have
.. code-block:: python
class OldName(SomeClass): ...
and we want to rename it to NewName, we can do the following:
.. code-block:: python
class NewName(SomeClass): ...
OldName = create_deprecated_class("OldName", NewName)
Then, if user class inherits from OldName, warning is issued. Also, if
some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)``
checks they'll still return True if sub is a subclass of NewName instead of
OldName.
"""
# https://github.com/python/mypy/issues/4177
class DeprecatedClass(new_class.__class__): # type: ignore[misc,name-defined]
# pylint: disable=no-self-argument
deprecated_class: type | None = None
warned_on_subclass: bool = False
def __new__( # pylint: disable=bad-classmethod-argument
metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]
) -> type:
cls: type = super().__new__(metacls, name, bases, clsdict_)
if metacls.deprecated_class is None:
metacls.deprecated_class = cls
return cls
def __init__(cls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]):
meta = cls.__class__
old = meta.deprecated_class
if old in bases and not (warn_once and meta.warned_on_subclass):
meta.warned_on_subclass = True
msg = subclass_warn_message.format(
cls=_clspath(cls),
old=_clspath(old, old_class_path),
new=_clspath(new_class, new_class_path),
)
if warn_once:
msg += " (warning only on first subclass, there may be others)"
warnings.warn(msg, warn_category, stacklevel=2)
super().__init__(name, bases, clsdict_)
# see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
# and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks
# for implementation details
def __instancecheck__(cls, inst: Any) -> bool:
return any(cls.__subclasscheck__(c) for c in (type(inst), inst.__class__))
def __subclasscheck__(cls, sub: type) -> bool:
if cls is not DeprecatedClass.deprecated_class:
# we should do the magic only if second `issubclass` argument
# is the deprecated class itself - subclasses of the
# deprecated class should not use custom `__subclasscheck__`
# method.
return cast("bool", super().__subclasscheck__(sub))
if not inspect.isclass(sub):
raise TypeError("issubclass() arg 1 must be a class")
mro = getattr(sub, "__mro__", ())
return any(c in {cls, new_class} for c in mro)
def __call__(cls, *args: Any, **kwargs: Any) -> Any:
old = DeprecatedClass.deprecated_class
if cls is old:
msg = instance_warn_message.format(
cls=_clspath(cls, old_class_path),
new=_clspath(new_class, new_class_path),
)
warnings.warn(msg, warn_category, stacklevel=2)
return super().__call__(*args, **kwargs)
deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})
try:
frm = inspect.stack()[1]
parent_module = inspect.getmodule(frm[0])
if parent_module is not None:
deprecated_cls.__module__ = parent_module.__name__
except Exception as e:
# Sometimes inspect.stack() fails (e.g. when the first import of
# deprecated class is in jinja2 template). __module__ attribute is not
# important enough to raise an exception as users may be unable
# to fix inspect.stack() errors.
warnings.warn(f"Error detecting parent module: {e!r}", stacklevel=2)
return deprecated_cls
def _clspath(cls: type, forced: str | None = None) -> str:
if forced is not None:
return forced
return f"{cls.__module__}.{cls.__name__}"
"""Return a deprecated alias of *new_class* named *name*."""
cls: type = deprecated_class(
name,
new_class,
namespace=clsdict,
category=warn_category,
warn_once=warn_once,
old_path=old_class_path,
new_path=new_class_path,
subclass_message=subclass_warn_message,
instance_message=instance_warn_message,
)
# deprecated_class() takes the module of the alias from its calling frame,
# which is this function and the decorator wrapping it, so skip past both.
frame = inspect.currentframe()
assert frame is not None
while frame.f_globals.get("__name__") in _WRAPPER_MODULES:
assert frame.f_back is not None
frame = frame.f_back
cls.__module__ = frame.f_globals.get("__name__", cls.__module__)
return cls
DEPRECATION_RULES: list[tuple[str, str]] = []

18
tests/test_utils_decorators.py
View File

@ -18,7 +18,12 @@ if TYPE_CHECKING:
class TestDeprecated:
def test_warns_and_still_calls(self):
@deprecated()
with pytest.warns(
ScrapyDeprecationWarning, match=r"decorators\.deprecated\(\) is deprecated"
):
decorate = deprecated()
@decorate
def add(a: int, b: int) -> int:
return a + b
@ -30,7 +35,11 @@ class TestDeprecated:
assert result == 5
def test_use_instead_in_message(self):
@deprecated(use_instead="other_function")
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
decorate = deprecated(use_instead="other_function")
@decorate
def old() -> None:
return None
@ -41,10 +50,13 @@ class TestDeprecated:
old()
def test_applied_without_parentheses(self):
@deprecated
def square(x: int) -> int:
return x * x
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
square = deprecated(square)
with pytest.warns(
ScrapyDeprecationWarning, match=r"Call to deprecated function square\."
) as record:

251
tests/test_utils_deprecate.py
View File

@ -1,256 +1,55 @@
from __future__ import annotations
import inspect
import warnings
from unittest import mock
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import create_deprecated_class, update_classpath
from scrapy.utils.deprecate import attribute, create_deprecated_class, update_classpath
class MyWarning(UserWarning):
class NewName:
pass
class SomeBaseClass:
pass
def test_attribute():
with pytest.warns(
ScrapyDeprecationWarning,
match=r"NewName\.old attribute is deprecated and will be no longer supported"
r" in Scrapy 1\.0, use NewName\.new attribute instead",
):
attribute(NewName(), "old", "new", version="1.0")
class NewName(SomeBaseClass):
pass
class TestWarnWhenSubclassed:
def test_no_warning_on_definition(self):
with warnings.catch_warnings():
warnings.simplefilter("error", category=ScrapyDeprecationWarning)
class TestCreateDeprecatedClass:
def test_warns_about_itself(self):
with pytest.warns(
ScrapyDeprecationWarning, match=r"create_deprecated_class\(\) is deprecated"
):
create_deprecated_class("Deprecated", NewName)
def test_subclassing_warning_message(self):
msg = (
r"tests\.test_utils_deprecate\.UserClass inherits from "
r"deprecated class tests\.test_utils_deprecate\.Deprecated, "
r"please inherit from tests\.test_utils_deprecate\.NewName."
r" \(warning only on first subclass, there may be others\)"
)
Deprecated = create_deprecated_class(
"Deprecated", NewName, warn_category=MyWarning
)
with pytest.warns(MyWarning, match=msg) as w:
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
assert w[0].lineno == inspect.getsourcelines(UserClass)[1]
def test_custom_class_paths(self):
Deprecated = create_deprecated_class(
"Deprecated",
NewName,
new_class_path="foo.NewClass",
old_class_path="bar.OldClass",
warn_category=MyWarning,
)
with pytest.warns(
MyWarning,
match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass",
):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with pytest.warns(
MyWarning,
match=r"bar\.OldClass is deprecated, instantiate foo\.NewClass instead",
):
_ = Deprecated()
def test_subclassing_warns_only_on_direct_children(self):
Deprecated = create_deprecated_class(
"Deprecated", NewName, warn_once=False, warn_category=MyWarning
)
with pytest.warns(
MyWarning,
match="UserClass inherits from deprecated class",
):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with warnings.catch_warnings():
warnings.simplefilter("error", MyWarning)
class NoWarnOnMe(UserClass):
pass
def test_subclassing_warns_once_by_default(self):
Deprecated = create_deprecated_class(
"Deprecated", NewName, warn_category=MyWarning
)
with pytest.warns(
MyWarning,
match="UserClass inherits from deprecated class",
):
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with warnings.catch_warnings():
warnings.simplefilter("error", MyWarning)
class FooClass(Deprecated): # type: ignore[misc, valid-type]
pass
class BarClass(Deprecated): # type: ignore[misc, valid-type]
pass
def test_warning_on_instance(self):
Deprecated = create_deprecated_class(
"Deprecated", NewName, warn_category=MyWarning
)
with pytest.warns(
MyWarning,
match=r"tests\.test_utils_deprecate\.Deprecated is deprecated, "
r"instantiate tests\.test_utils_deprecate\.NewName instead\.",
) as w:
_, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) # type: ignore[arg-type]
assert len(w) == 1
assert w[0].lineno == lineno
# ignore subclassing warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore", MyWarning)
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
with warnings.catch_warnings():
warnings.simplefilter("error", MyWarning)
UserClass() # subclass instances don't warn
def test_warning_auto_message(self):
Deprecated = create_deprecated_class("Deprecated", NewName)
with pytest.warns(
ScrapyDeprecationWarning,
match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName",
):
class UserClass2(Deprecated): # type: ignore[misc, valid-type]
pass
def test_issubclass(self):
def test_returns_a_working_alias(self):
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
DeprecatedName = create_deprecated_class("DeprecatedName", NewName)
class UpdatedUserClass1(NewName):
pass
class UpdatedUserClass1a(NewName):
pass
class OutdatedUserClass1(DeprecatedName): # type: ignore[misc, valid-type]
pass
class OutdatedUserClass1a(DeprecatedName): # type: ignore[misc, valid-type]
pass
class UnrelatedClass:
pass
assert issubclass(UpdatedUserClass1, NewName)
assert issubclass(UpdatedUserClass1a, NewName)
assert issubclass(UpdatedUserClass1, DeprecatedName)
assert issubclass(UpdatedUserClass1a, DeprecatedName)
assert issubclass(OutdatedUserClass1, DeprecatedName)
assert not issubclass(UnrelatedClass, DeprecatedName)
assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a)
assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)
with pytest.raises(TypeError):
issubclass(object(), DeprecatedName) # type: ignore[arg-type]
def test_isinstance(self):
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
DeprecatedName = create_deprecated_class("DeprecatedName", NewName)
class UpdatedUserClass2(NewName):
pass
class UpdatedUserClass2a(NewName):
pass
class OutdatedUserClass2(DeprecatedName): # type: ignore[misc, valid-type]
pass
class OutdatedUserClass2a(DeprecatedName): # type: ignore[misc, valid-type]
pass
class UnrelatedClass:
pass
assert isinstance(UpdatedUserClass2(), NewName)
assert isinstance(UpdatedUserClass2a(), NewName)
assert isinstance(UpdatedUserClass2(), DeprecatedName)
assert isinstance(UpdatedUserClass2a(), DeprecatedName)
assert isinstance(OutdatedUserClass2(), DeprecatedName)
assert isinstance(OutdatedUserClass2a(), DeprecatedName)
assert not isinstance(OutdatedUserClass2a(), OutdatedUserClass2)
assert not isinstance(OutdatedUserClass2(), OutdatedUserClass2a)
assert not isinstance(UnrelatedClass(), DeprecatedName)
def test_clsdict(self):
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"})
Deprecated = create_deprecated_class(
"Deprecated", NewName, {"foo": "bar"}, warn_once=False
)
assert Deprecated.__module__ == __name__
assert Deprecated.foo == "bar" # type: ignore[attr-defined]
def test_deprecate_a_class_with_custom_metaclass(self):
Meta1 = type("Meta1", (type,), {})
New = Meta1("New", (), {})
create_deprecated_class("Deprecated", New)
def test_deprecate_subclass_of_deprecated_class(self):
with warnings.catch_warnings():
warnings.simplefilter("error", MyWarning)
Deprecated = create_deprecated_class(
"Deprecated", NewName, warn_category=MyWarning
)
AlsoDeprecated = create_deprecated_class(
"AlsoDeprecated",
Deprecated,
new_class_path="foo.Bar",
warn_category=MyWarning,
)
with pytest.warns(
MyWarning,
match=r"AlsoDeprecated is deprecated, instantiate foo\.Bar instead",
):
AlsoDeprecated()
with pytest.warns(
MyWarning,
match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar",
ScrapyDeprecationWarning,
match=r"tests\.test_utils_deprecate\.UserClass inherits from deprecated"
r" class tests\.test_utils_deprecate\.Deprecated, please inherit from"
r" tests\.test_utils_deprecate\.NewName\.",
):
class UserClass(AlsoDeprecated): # type: ignore[misc, valid-type]
class UserClass(Deprecated): # type: ignore[misc, valid-type]
pass
def test_inspect_stack(self):
with (
mock.patch("inspect.stack", side_effect=IndexError),
pytest.warns(UserWarning, match="Error detecting parent module"),
):
create_deprecated_class("DeprecatedName", NewName)
assert issubclass(UserClass, Deprecated)
@mock.patch(

2
tox.ini
View File

@ -142,6 +142,7 @@ deps =
brotlicffi==1.2.0.0; implementation_name == "pypy"
cryptography==37.0.0
cssselect==0.9.1
formerly==0.1.0
httpx2==2.0.0
itemadapter==0.1.0
lxml==4.6.4
@ -149,6 +150,7 @@ deps =
pyOpenSSL==22.0.0
queuelib==1.6.1
service_identity==23.1.0
typing-extensions==4.5.0
w3lib==1.17.0
zope.interface==5.1.0
{[test-requirements]deps}