RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove code deprecated in 2.12.0 (#7126) 

* Remove smaller things.

* Remove from_settings().

* Remove old MediaPipeline init code.

* Cleanup.

* Typo.
This commit is contained in:
Andrey Rakhmatullin 2025-10-27 17:14:38 +05:00 committed by GitHub
parent d414d393d4
commit 61b4befc60
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 48 additions and 854 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/conf.py
View File

@ -126,7 +126,7 @@ coverage_ignore_pyobjects = [
# The interface methods of duplicate request filtering classes are already
# covered in the interface documentation part of the DUPEFILTER_CLASS
# setting documentation.
r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_settings|request_seen|open|close|log)$",
r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_crawler|request_seen|open|close|log)$",
# Private exception used by the command-line interface implementation.
r"^scrapy\.exceptions\.UsageError",
# Methods of BaseItemExporter subclasses are only documented in

4
docs/topics/settings.rst
View File

@ -1131,9 +1131,9 @@ interface::
class MyDupeFilter:
@classmethod
def from_settings(cls, settings):
def from_crawler(cls, crawler):
"""Returns an instance of this duplicate request filtering class
based on the current crawl settings."""
based on the current Crawler instance."""
return cls()
def request_seen(self, request):

17
scrapy/__init__.py
View File

@ -29,23 +29,6 @@ __version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."))
def __getattr__(name: str):
if name == "twisted_version":
import warnings # noqa: PLC0415 # pylint: disable=reimported
from twisted import version as _txv # noqa: PLC0415
from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415
warnings.warn(
"The scrapy.twisted_version attribute is deprecated, use twisted.version instead",
ScrapyDeprecationWarning,
)
return _txv.major, _txv.minor, _txv.micro
raise AttributeError
# Ignore noisy twisted deprecation warnings
warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted")

8
scrapy/core/downloader/__init__.py
View File

@ -182,14 +182,6 @@ class Downloader:
return key
def _get_slot_key(self, request: Request, spider: Spider | None) -> str:
warnings.warn(
"Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return self.get_slot_key(request)
# passed as download_func into self.middleware.download() in self.fetch()
@inlineCallbacks
def _enqueue_request(

29
scrapy/core/downloader/contextfactory.py
View File

@ -71,21 +71,6 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
stacklevel=2,
)
@classmethod
def from_settings(
cls,
settings: BaseSettings,
method: int = SSL.SSLv23_METHOD,
*args: Any,
**kwargs: Any,
) -> Self:
warnings.warn(
f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return cls._from_settings(settings, method, *args, **kwargs)
@classmethod
def from_crawler(
cls,
@ -94,20 +79,10 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
*args: Any,
**kwargs: Any,
) -> Self:
return cls._from_settings(crawler.settings, method, *args, **kwargs)
@classmethod
def _from_settings(
cls,
settings: BaseSettings,
method: int = SSL.SSLv23_METHOD,
*args: Any,
**kwargs: Any,
) -> Self:
tls_verbose_logging: bool = settings.getbool(
tls_verbose_logging: bool = crawler.settings.getbool(
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
)
tls_ciphers: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
tls_ciphers: str | None = crawler.settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
return cls( # type: ignore[misc]
method=method,
tls_verbose_logging=tls_verbose_logging,

41
scrapy/dupefilters.py
View File

@ -1,7 +1,6 @@
from __future__ import annotations
import logging
import warnings
from pathlib import Path
from typing import TYPE_CHECKING
from warnings import warn
@ -22,7 +21,6 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
@ -30,15 +28,6 @@ class BaseDupeFilter:
"""Dummy duplicate request filtering class (:setting:`DUPEFILTER_CLASS`)
that does not filter out any request."""
@classmethod
def from_settings(cls, settings: BaseSettings) -> Self:
warnings.warn(
f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return cls()
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
return cls()
@ -88,38 +77,16 @@ class RFPDupeFilter(BaseDupeFilter):
self.file.seek(0)
self.fingerprints.update(x.rstrip() for x in self.file)
@classmethod
def from_settings(
cls,
settings: BaseSettings,
*,
fingerprinter: RequestFingerprinterProtocol | None = None,
) -> Self:
warnings.warn(
f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return cls._from_settings(settings, fingerprinter=fingerprinter)
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
assert crawler.request_fingerprinter
return cls._from_settings(
crawler.settings,
debug = crawler.settings.getbool("DUPEFILTER_DEBUG")
return cls(
job_dir(crawler.settings),
debug,
fingerprinter=crawler.request_fingerprinter,
)
@classmethod
def _from_settings(
cls,
settings: BaseSettings,
*,
fingerprinter: RequestFingerprinterProtocol | None = None,
) -> Self:
debug = settings.getbool("DUPEFILTER_DEBUG")
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
def request_seen(self, request: Request) -> bool:
fp = self.request_fingerprint(request)
if fp in self.fingerprints:

23
scrapy/extensions/feedexport.py
View File

@ -16,7 +16,7 @@ from collections.abc import Callable
from datetime import datetime, timezone
from pathlib import Path, PureWindowsPath
from tempfile import NamedTemporaryFile
from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, TypeVar, cast
from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, cast
from urllib.parse import unquote, urlparse
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
@ -35,8 +35,6 @@ from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
from collections.abc import Iterable
from _typeshed import OpenBinaryMode
from twisted.python.failure import Failure
@ -54,25 +52,6 @@ UriParamsCallableT: TypeAlias = Callable[
[dict[str, Any], Spider], dict[str, Any] | None
]
_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")
def build_storage(
builder: Callable[..., _StorageT],
uri: str,
*args: Any,
feed_options: dict[str, Any] | None = None,
preargs: Iterable[Any] = (),
**kwargs: Any,
) -> _StorageT:
warnings.warn(
"scrapy.extensions.feedexport.build_storage() is deprecated, call the builder directly.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
kwargs["feed_options"] = feed_options
return builder(*preargs, uri, *args, **kwargs)
class ItemFilter:
"""

17
scrapy/mail.py
View File

@ -7,7 +7,6 @@ See documentation in docs/topics/email.rst
from __future__ import annotations
import logging
import warnings
from email import encoders as Encoders
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
@ -20,7 +19,6 @@ from typing import IO, TYPE_CHECKING, Any
from twisted.internet import ssl
from twisted.internet.defer import Deferred
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
@ -35,7 +33,6 @@ if TYPE_CHECKING:
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
logger = logging.getLogger(__name__)
@ -73,21 +70,9 @@ class MailSender:
self.mailfrom: str = mailfrom
self.debug: bool = debug
@classmethod
def from_settings(cls, settings: BaseSettings) -> Self:
warnings.warn(
f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return cls._from_settings(settings)
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
return cls._from_settings(crawler.settings)
@classmethod
def _from_settings(cls, settings: BaseSettings) -> Self:
settings = crawler.settings
return cls(
smtphost=settings["MAIL_HOST"],
mailfrom=settings["MAIL_FROM"],

34
scrapy/middleware.py
View File

@ -23,7 +23,7 @@ if TYPE_CHECKING:
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings, Settings
from scrapy.settings import Settings
logger = logging.getLogger(__name__)
@ -100,43 +100,15 @@ class MiddlewareManager(ABC):
def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]:
raise NotImplementedError
@staticmethod
def _build_from_settings(objcls: type[_T], settings: BaseSettings) -> _T:
if hasattr(objcls, "from_settings"):
instance = objcls.from_settings(settings) # type: ignore[attr-defined]
method_name = "from_settings"
else:
instance = objcls()
method_name = "__new__"
if instance is None:
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
return cast("_T", instance)
@classmethod
def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:
warnings.warn(
f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return cls._from_settings(settings, crawler)
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
return cls._from_settings(crawler.settings, crawler)
@classmethod
def _from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:
mwlist = cls._get_mwlist_from_settings(settings)
mwlist = cls._get_mwlist_from_settings(crawler.settings)
middlewares = []
enabled = []
for clspath in mwlist:
try:
mwcls = load_object(clspath)
if crawler is not None:
mw = build_from_crawler(mwcls, crawler)
else:
mw = MiddlewareManager._build_from_settings(mwcls, settings)
mw = build_from_crawler(mwcls, crawler)
middlewares.append(mw)
enabled.append(clspath)
except NotConfigured as e:

71
scrapy/pipelines/files.py
View File

@ -12,7 +12,6 @@ import hashlib
import logging
import mimetypes
import time
import warnings
from collections import defaultdict
from contextlib import suppress
from ftplib import FTP
@ -25,17 +24,15 @@ from itemadapter import ItemAdapter
from twisted.internet.defer import Deferred, maybeDeferred
from twisted.internet.threads import deferToThread
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.datatypes import CaseInsensitiveDict
from scrapy.utils.deprecate import method_is_overridden
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.python import get_func_args, global_object_name, to_bytes
from scrapy.utils.python import to_bytes
from scrapy.utils.request import referer_str
if TYPE_CHECKING:
@ -49,6 +46,7 @@ if TYPE_CHECKING:
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
logger = logging.getLogger(__name__)
@ -161,7 +159,7 @@ class S3FilesStore:
AWS_USE_SSL = None
AWS_VERIFY = None
POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings
POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler()
HEADERS = {
"Cache-Control": "max-age=172800",
}
@ -280,7 +278,7 @@ class GCSFilesStore:
CACHE_CONTROL = "max-age=172800"
# The bucket's default object ACL will be applied to the object.
# Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
# Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_crawler().
POLICY = None
def __init__(self, uri: str):
@ -446,9 +444,8 @@ class FilesPipeline(MediaPipeline):
self,
store_uri: str | PathLike[str],
download_func: Callable[[Request, Spider], Response] | None = None,
settings: Settings | dict[str, Any] | None = None,
*,
crawler: Crawler | None = None,
crawler: Crawler,
):
if not (store_uri and (store_uri := _to_string(store_uri))):
from scrapy.pipelines.images import ImagesPipeline # noqa: PLC0415
@ -461,18 +458,7 @@ class FilesPipeline(MediaPipeline):
f"to enable {self.__class__.__name__}."
)
if crawler is not None:
if settings is not None:
warnings.warn(
f"FilesPipeline.__init__() was called with a crawler instance and a settings instance"
f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored"
f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
settings = crawler.settings
elif isinstance(settings, dict) or settings is None:
settings = Settings(settings)
settings = crawler.settings
cls_name = "FilesPipeline"
self.store: FilesStoreProtocol = self._get_store(store_uri)
resolve = functools.partial(
@ -490,51 +476,14 @@ class FilesPipeline(MediaPipeline):
resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD
)
super().__init__(
download_func=download_func,
settings=settings if not crawler else None,
crawler=crawler,
)
@classmethod
def from_settings(cls, settings: Settings) -> Self:
warnings.warn(
f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return cls._from_settings(settings, None)
super().__init__(download_func=download_func, crawler=crawler)
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
if method_is_overridden(cls, FilesPipeline, "from_settings"):
warnings.warn(
f"{global_object_name(cls)} overrides FilesPipeline.from_settings()."
f" This method is deprecated and won't be called in future Scrapy versions,"
f" please update your code so that it overrides from_crawler() instead.",
category=ScrapyDeprecationWarning,
)
o = cls.from_settings(crawler.settings)
o._finish_init(crawler)
return o
return cls._from_settings(crawler.settings, crawler)
@classmethod
def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self:
settings = crawler.settings
cls._update_stores(settings)
store_uri = settings["FILES_STORE"]
if "crawler" in get_func_args(cls.__init__):
o = cls(store_uri, crawler=crawler)
else:
o = cls(store_uri, settings=settings)
if crawler:
o._finish_init(crawler)
warnings.warn(
f"{global_object_name(cls)}.__init__() doesn't take a crawler argument."
" This is deprecated and the argument will be required in future Scrapy versions.",
category=ScrapyDeprecationWarning,
)
return o
return cls(store_uri, crawler=crawler)
@classmethod
def _update_stores(cls, settings: BaseSettings) -> None:

46
scrapy/pipelines/images.py
View File

@ -8,19 +8,17 @@ from __future__ import annotations
import functools
import hashlib
import warnings
from contextlib import suppress
from io import BytesIO
from typing import TYPE_CHECKING, Any
from itemadapter import ItemAdapter
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum
from scrapy.settings import Settings
from scrapy.utils.python import get_func_args, global_object_name, to_bytes
from scrapy.utils.python import to_bytes
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
@ -58,9 +56,8 @@ class ImagesPipeline(FilesPipeline):
self,
store_uri: str | PathLike[str],
download_func: Callable[[Request, Spider], Response] | None = None,
settings: Settings | dict[str, Any] | None = None,
*,
crawler: Crawler | None = None,
crawler: Crawler,
):
try:
from PIL import Image, ImageOps # noqa: PLC0415
@ -72,26 +69,9 @@ class ImagesPipeline(FilesPipeline):
"ImagesPipeline requires installing Pillow 8.3.2 or later"
)
super().__init__(
store_uri,
settings=settings if not crawler else None,
download_func=download_func,
crawler=crawler,
)
if crawler is not None:
if settings is not None:
warnings.warn(
f"ImagesPipeline.__init__() was called with a crawler instance and a settings instance"
f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored"
f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
settings = crawler.settings
elif isinstance(settings, dict) or settings is None:
settings = Settings(settings)
super().__init__(store_uri, download_func=download_func, crawler=crawler)
settings = crawler.settings
resolve = functools.partial(
self._key_for_pipe,
base_class_name="ImagesPipeline",
@ -121,21 +101,11 @@ class ImagesPipeline(FilesPipeline):
)
@classmethod
def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self:
def from_crawler(cls, crawler: Crawler) -> Self:
settings = crawler.settings
cls._update_stores(settings)
store_uri = settings["IMAGES_STORE"]
if "crawler" in get_func_args(cls.__init__):
o = cls(store_uri, crawler=crawler)
else:
o = cls(store_uri, settings=settings)
if crawler:
o._finish_init(crawler)
warnings.warn(
f"{global_object_name(cls)}.__init__() doesn't take a crawler argument."
" This is deprecated and the argument will be required in future Scrapy versions.",
category=ScrapyDeprecationWarning,
)
return o
return cls(store_uri, crawler=crawler)
def file_downloaded(
self,

75
scrapy/pipelines/media.py
View File

@ -19,14 +19,13 @@ from twisted.python.versions import Version
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http.request import NO_CALLBACK, Request
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import _DEFER_DELAY, _defer_sleep, deferred_from_coro
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import get_func_args, global_object_name
from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
from collections.abc import Callable, Generator
@ -37,6 +36,7 @@ if TYPE_CHECKING:
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.settings import Settings
from scrapy.utils.request import RequestFingerprinterProtocol
@ -55,10 +55,6 @@ logger = logging.getLogger(__name__)
class MediaPipeline(ABC):
crawler: Crawler
_fingerprinter: RequestFingerprinterProtocol
_modern_init = False
LOG_FAILED_RESULTS: bool = True
class SpiderInfo:
@ -73,24 +69,17 @@ class MediaPipeline(ABC):
def __init__(
self,
download_func: Callable[[Request, Spider], Response] | None = None,
settings: Settings | dict[str, Any] | None = None,
*,
crawler: Crawler | None = None,
crawler: Crawler,
):
self.crawler: Crawler = crawler
assert crawler.request_fingerprinter
self._fingerprinter: RequestFingerprinterProtocol = (
crawler.request_fingerprinter
)
self.download_func = download_func
if crawler is not None:
if settings is not None:
warnings.warn(
f"MediaPipeline.__init__() was called with a crawler instance and a settings instance"
f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored"
f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
settings = crawler.settings
elif isinstance(settings, dict) or settings is None:
settings = Settings(settings)
settings = crawler.settings
resolve = functools.partial(
self._key_for_pipe, base_class_name="MediaPipeline", settings=settings
)
@ -99,27 +88,6 @@ class MediaPipeline(ABC):
)
self._handle_statuses(self.allow_redirects)
if crawler:
self._finish_init(crawler)
self._modern_init = True
else:
warnings.warn(
f"MediaPipeline.__init__() was called without the crawler argument"
f" when creating {global_object_name(self.__class__)}."
f" This is deprecated and the argument will be required in future Scrapy versions.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
def _finish_init(self, crawler: Crawler) -> None:
# This was done in from_crawler() before 2.12, now it's done in __init__()
# if the crawler was passed to it and may be needed to be called in other
# deprecated code paths explicitly too. After the crawler argument of __init__()
# becomes mandatory this should be inlined there.
self.crawler = crawler
assert crawler.request_fingerprinter
self._fingerprinter = crawler.request_fingerprinter
def _handle_statuses(self, allow_redirects: bool) -> None:
self.handle_httpstatus_list = None
if allow_redirects:
@ -143,30 +111,7 @@ class MediaPipeline(ABC):
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
pipe: Self
if hasattr(cls, "from_settings"):
pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined]
warnings.warn(
f"{global_object_name(cls)} has from_settings() and either doesn't have"
" from_crawler() or calls MediaPipeline.from_crawler() from it,"
" so from_settings() was used to create the instance of it."
" This is deprecated and calling from_settings() will be removed"
" in a future Scrapy version. Please move the initialization code into"
" from_crawler() or __init__().",
category=ScrapyDeprecationWarning,
)
elif "crawler" in get_func_args(cls.__init__):
pipe = cls(crawler=crawler)
else:
pipe = cls()
warnings.warn(
f"{global_object_name(cls)}.__init__() doesn't take a crawler argument."
" This is deprecated and the argument will be required in future Scrapy versions.",
category=ScrapyDeprecationWarning,
)
if not pipe._modern_init:
pipe._finish_init(crawler)
return pipe
return cls(crawler=crawler)
@_warn_spider_arg
def open_spider(self, spider: Spider | None = None) -> None:

2
scrapy/settings/default_settings.py
View File

@ -152,7 +152,6 @@ __all__ = [
"REFERER_ENABLED",
"REFERRER_POLICY",
"REQUEST_FINGERPRINTER_CLASS",
"REQUEST_FINGERPRINTER_IMPLEMENTATION",
"RETRY_ENABLED",
"RETRY_EXCEPTIONS",
"RETRY_HTTP_CODES",
@ -451,7 +450,6 @@ REFERER_ENABLED = True
REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL"
RETRY_ENABLED = True
RETRY_EXCEPTIONS = [

25
scrapy/utils/defer.py
View File

@ -317,31 +317,6 @@ def process_chain(
return d
def process_chain_both(
callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]],
input: Any, # noqa: A002
*a: _P.args,
**kw: _P.kwargs,
) -> Deferred:
"""Return a Deferred built by chaining the given callbacks and errbacks"""
warnings.warn(
"process_chain_both() is deprecated and will be removed in a future"
" Scrapy version.",
ScrapyDeprecationWarning,
stacklevel=2,
)
d: Deferred = Deferred()
for cb, eb in zip(callbacks, errbacks, strict=False):
d.addCallback(cb, *a, **kw)
d.addErrback(eb, *a, **kw)
if isinstance(input, failure.Failure):
d.errback(input)
else:
d.callback(input)
return d
def process_parallel(
callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]],
input: _T, # noqa: A002

55
scrapy/utils/misc.py
View File

@ -129,52 +129,10 @@ def rel_has_nofollow(rel: str | None) -> bool:
return rel is not None and "nofollow" in rel.replace(",", " ").split()
def create_instance(objcls, settings, crawler, *args, **kwargs):
"""Construct a class instance using its ``from_crawler`` or
``from_settings`` constructors, if available.
At least one of ``settings`` and ``crawler`` needs to be different from
``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
tried.
``*args`` and ``**kwargs`` are forwarded to the constructors.
Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
.. versionchanged:: 2.2
Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
extension has not been implemented correctly).
"""
warnings.warn(
"The create_instance() function is deprecated. "
"Please use build_from_crawler() instead.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if settings is None:
if crawler is None:
raise ValueError("Specify at least one of settings and crawler.")
settings = crawler.settings
if crawler and hasattr(objcls, "from_crawler"):
instance = objcls.from_crawler(crawler, *args, **kwargs)
method_name = "from_crawler"
elif hasattr(objcls, "from_settings"):
instance = objcls.from_settings(settings, *args, **kwargs)
method_name = "from_settings"
else:
instance = objcls(*args, **kwargs)
method_name = "__new__"
if instance is None:
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
return instance
def build_from_crawler(
objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any
) -> T:
"""Construct a class instance using its ``from_crawler`` or ``from_settings`` constructor.
"""Construct a class instance using its ``from_crawler()`` or ``__init__()`` constructor.
.. versionadded:: 2.12
@ -185,17 +143,6 @@ def build_from_crawler(
if hasattr(objcls, "from_crawler"):
instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined]
method_name = "from_crawler"
elif hasattr(objcls, "from_settings"):
warnings.warn(
f"{objcls.__qualname__} has from_settings() but not from_crawler()."
" This is deprecated and calling from_settings() will be removed in a future"
" Scrapy version. You can implement a simple from_crawler() that calls"
" from_settings() with crawler.settings.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
instance = objcls.from_settings(crawler.settings, *args, **kwargs) # type: ignore[attr-defined]
method_name = "from_settings"
else:
instance = objcls(*args, **kwargs)
method_name = "__new__"

68
scrapy/utils/python.py
View File

@ -8,14 +8,12 @@ import gc
import inspect
import re
import sys
import warnings
import weakref
from collections.abc import AsyncIterator, Iterable, Mapping
from functools import partial, wraps
from itertools import chain
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncgen import as_async_generator
if TYPE_CHECKING:
@ -32,47 +30,6 @@ _VT = TypeVar("_VT")
_P = ParamSpec("_P")
def flatten(x: Iterable[Any]) -> list[Any]:
"""flatten(sequence) -> list
Returns a single, flat list which contains all elements retrieved
from the sequence and all recursively contained sub-sequences
(iterables).
Examples:
>>> [1, 2, [3,4], (5,6)]
[1, 2, [3, 4], (5, 6)]
>>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)])
[1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]
>>> flatten(["foo", "bar"])
['foo', 'bar']
>>> flatten(["foo", ["baz", 42], "bar"])
['foo', 'baz', 42, 'bar']
"""
warnings.warn(
"The flatten function is deprecated and will be removed in a future version of Scrapy.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return list(iflatten(x))
def iflatten(x: Iterable[Any]) -> Iterable[Any]:
"""iflatten(sequence) -> iterator
Similar to ``.flatten()``, but returns iterator instead"""
warnings.warn(
"The iflatten function is deprecated and will be removed in a future version of Scrapy.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
for el in x:
if is_listlike(el):
yield from iflatten(el)
else:
yield el
def is_listlike(x: Any) -> bool:
"""
>>> is_listlike("foo")
@ -289,31 +246,6 @@ def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]:
return args, kwargs
def equal_attributes(
obj1: Any, obj2: Any, attributes: list[str | Callable[[Any], Any]] | None
) -> bool:
"""Compare two objects attributes"""
warnings.warn(
"The equal_attributes function is deprecated and will be removed in a future version of Scrapy.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
# not attributes given return False by default
if not attributes:
return False
temp1, temp2 = object(), object()
for attr in attributes:
# support callables like itemgetter
if callable(attr):
if attr(obj1) != attr(obj2):
return False
elif getattr(obj1, attr, temp1) != getattr(obj2, attr, temp2):
return False
# all attributes equal
return True
@overload
def without_none_values(iterable: Mapping[_KT, _VT]) -> dict[_KT, _VT]: ...

32
scrapy/utils/request.py
View File

@ -7,16 +7,13 @@ from __future__ import annotations
import hashlib
import json
import warnings
from typing import TYPE_CHECKING, Any, Protocol
from urllib.parse import urlunparse
from weakref import WeakKeyDictionary
from w3lib.http import basic_auth_header
from w3lib.url import canonicalize_url
from scrapy import Request, Spider
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_bytes, to_unicode
@ -120,41 +117,12 @@ class RequestFingerprinter:
return cls(crawler)
def __init__(self, crawler: Crawler | None = None):
if crawler:
implementation = crawler.settings.get(
"REQUEST_FINGERPRINTER_IMPLEMENTATION"
)
else:
implementation = "SENTINEL"
if implementation != "SENTINEL":
message = (
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"
"It will be removed in a future version of Scrapy."
)
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
self._fingerprint = fingerprint
def fingerprint(self, request: Request) -> bytes:
return self._fingerprint(request)
def request_authenticate(
request: Request,
username: str,
password: str,
) -> None:
"""Authenticate the given request (in place) using the HTTP basic access
authentication mechanism (RFC 2617) and the given username and password
"""
warnings.warn(
"The request_authenticate function is deprecated and will be removed in a future version of Scrapy.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
request.headers["Authorization"] = basic_auth_header(username, password)
def request_httprepr(request: Request) -> bytes:
"""Return the raw HTTP representation (as bytes) of the given request.
This is provided only for reference since it's not the actual stream of

12
scrapy/utils/serialize.py
View File

@ -1,13 +1,11 @@
import datetime
import decimal
import json
import warnings
from typing import Any
from itemadapter import ItemAdapter, is_item
from twisted.internet import defer
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
@ -35,13 +33,3 @@ class ScrapyJSONEncoder(json.JSONEncoder):
if is_item(o):
return ItemAdapter(o).asdict()
return super().default(o)
class ScrapyJSONDecoder(json.JSONDecoder):
def __init__(self, *args, **kwargs):
warnings.warn(
"The ScrapyJSONDecoder class is deprecated and will be removed in a future version of Scrapy.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
super().__init__(*args, **kwargs)

16
scrapy/utils/test.py
View File

@ -12,7 +12,7 @@ from importlib import import_module
from pathlib import Path
from posixpath import split
from typing import TYPE_CHECKING, Any, TypeVar, cast
from unittest import TestCase, mock
from unittest import mock
from twisted.trial.unittest import SkipTest
from twisted.web.client import Agent
@ -166,20 +166,6 @@ def get_testenv() -> dict[str, str]:
return env
def assert_samelines(
testcase: TestCase, text1: str, text2: str, msg: str | None = None
) -> None:
"""Asserts text1 and text2 have the same lines, ignoring differences in
line endings between platforms
"""
warnings.warn(
"The assert_samelines function is deprecated and will be removed in a future version of Scrapy.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) # noqa: PT009
def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:
q: asyncio.Queue[_T] = asyncio.Queue()
getter = q.get()

31
tests/test_pipeline_files.py
View File

@ -756,37 +756,6 @@ class TestBuildFromCrawler:
assert len(w) == 0
assert pipe.store
def test_has_old_init(self):
class Pipeline(FilesPipeline):
def __init__(self, store_uri, download_func=None, settings=None):
super().__init__(store_uri, download_func, settings)
self._init_called = True
with warnings.catch_warnings(record=True) as w:
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert len(w) == 2
assert pipe._init_called
def test_has_from_settings(self):
class Pipeline(FilesPipeline):
_from_settings_called = False
@classmethod
def from_settings(cls, settings):
o = super().from_settings(settings)
o._from_settings_called = True
return o
with warnings.catch_warnings(record=True) as w:
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert len(w) == 3
assert pipe.store
assert pipe._from_settings_called
def test_has_from_crawler_and_init(self):
class Pipeline(FilesPipeline):
_from_crawler_called = False

78
tests/test_pipeline_media.py
View File

@ -418,84 +418,6 @@ class TestBuildFromCrawler:
assert pipe._fingerprinter
assert len(w) == 0
def test_has_old_init(self):
class Pipeline(UserDefinedPipeline):
def __init__(self):
super().__init__()
self._init_called = True
with warnings.catch_warnings(record=True) as w:
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert len(w) == 2
assert pipe._init_called
def test_has_from_settings(self):
class Pipeline(UserDefinedPipeline):
_from_settings_called = False
@classmethod
def from_settings(cls, settings):
o = cls()
o._from_settings_called = True
return o
with warnings.catch_warnings(record=True) as w:
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert len(w) == 2
assert pipe._from_settings_called
def test_has_from_settings_and_from_crawler(self):
class Pipeline(UserDefinedPipeline):
_from_settings_called = False
_from_crawler_called = False
@classmethod
def from_settings(cls, settings):
o = cls()
o._from_settings_called = True
return o
@classmethod
def from_crawler(cls, crawler):
o = super().from_crawler(crawler)
o._from_crawler_called = True
return o
with warnings.catch_warnings(record=True) as w:
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert len(w) == 2
assert pipe._from_settings_called
assert pipe._from_crawler_called
def test_has_from_settings_and_init(self):
class Pipeline(UserDefinedPipeline):
_from_settings_called = False
def __init__(self, store_uri, settings):
super().__init__()
self._init_called = True
@classmethod
def from_settings(cls, settings):
store_uri = settings["FILES_STORE"]
o = cls(store_uri, settings=settings)
o._from_settings_called = True
return o
with warnings.catch_warnings(record=True) as w:
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert len(w) == 2
assert pipe._from_settings_called
assert pipe._init_called
def test_has_from_crawler_and_init(self):
class Pipeline(UserDefinedPipeline):
_from_crawler_called = False

14
tests/test_scrapy__getattr__.py
View File

@ -1,20 +1,6 @@
import warnings
def test_deprecated_twisted_version():
with warnings.catch_warnings(record=True) as warns:
from scrapy import ( # noqa: PLC0415 # pylint: disable=no-name-in-module
twisted_version,
)
assert twisted_version is not None
assert isinstance(twisted_version, tuple)
assert (
"The scrapy.twisted_version attribute is deprecated, use twisted.version instead"
in warns[0].message.args
)
def test_deprecated_concurrent_requests_per_ip_attribute():
with warnings.catch_warnings(record=True) as warns:
from scrapy.settings.default_settings import ( # noqa: PLC0415

76
tests/test_utils_misc/__init__.py
View File

@ -9,7 +9,6 @@ from scrapy.item import Field, Item
from scrapy.utils.misc import (
arg_to_iter,
build_from_crawler,
create_instance,
load_object,
rel_has_nofollow,
set_environ,
@ -97,98 +96,29 @@ class TestUtilsMisc:
assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_create_instance(self):
settings = mock.MagicMock()
crawler = mock.MagicMock(spec_set=["settings"])
args = (True, 100.0)
kwargs = {"key": "val"}
def _test_with_settings(mock, settings):
create_instance(mock, settings, None, *args, **kwargs)
if hasattr(mock, "from_crawler"):
assert mock.from_crawler.call_count == 0
if hasattr(mock, "from_settings"):
mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
assert mock.call_count == 0
else:
mock.assert_called_once_with(*args, **kwargs)
def _test_with_crawler(mock, settings, crawler):
create_instance(mock, settings, crawler, *args, **kwargs)
if hasattr(mock, "from_crawler"):
mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
if hasattr(mock, "from_settings"):
assert mock.from_settings.call_count == 0
assert mock.call_count == 0
elif hasattr(mock, "from_settings"):
mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
assert mock.call_count == 0
else:
mock.assert_called_once_with(*args, **kwargs)
# Check usage of correct constructor using four mocks:
# 1. with no alternative constructors
# 2. with from_settings() constructor
# 3. with from_crawler() constructor
# 4. with from_settings() and from_crawler() constructor
spec_sets = (
["__qualname__"],
["__qualname__", "from_settings"],
["__qualname__", "from_crawler"],
["__qualname__", "from_settings", "from_crawler"],
)
for specs in spec_sets:
m = mock.MagicMock(spec_set=specs)
_test_with_settings(m, settings)
m.reset_mock()
_test_with_crawler(m, settings, crawler)
# Check adoption of crawler settings
m = mock.MagicMock(spec_set=["__qualname__", "from_settings"])
create_instance(m, None, crawler, *args, **kwargs)
m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs)
with pytest.raises(
ValueError, match="Specify at least one of settings and crawler"
):
create_instance(m, None, None)
m.from_settings.return_value = None
with pytest.raises(TypeError):
create_instance(m, settings, None)
def test_build_from_crawler(self):
settings = mock.MagicMock()
crawler = mock.MagicMock(spec_set=["settings"])
args = (True, 100.0)
kwargs = {"key": "val"}
def _test_with_crawler(mock, settings, crawler):
def _test_with_crawler(mock, crawler):
build_from_crawler(mock, crawler, *args, **kwargs)
if hasattr(mock, "from_crawler"):
mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
if hasattr(mock, "from_settings"):
assert mock.from_settings.call_count == 0
assert mock.call_count == 0
elif hasattr(mock, "from_settings"):
mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
assert mock.call_count == 0
else:
mock.assert_called_once_with(*args, **kwargs)
# Check usage of correct constructor using three mocks:
# Check usage of correct constructor using 2 mocks:
# 1. with no alternative constructors
# 2. with from_crawler() constructor
# 3. with from_settings() and from_crawler() constructor
spec_sets = (
["__qualname__"],
["__qualname__", "from_crawler"],
["__qualname__", "from_settings", "from_crawler"],
)
for specs in spec_sets:
m = mock.MagicMock(spec_set=specs)
_test_with_crawler(m, settings, crawler)
_test_with_crawler(m, crawler)
m.reset_mock()
# Check adoption of crawler

51
tests/test_utils_python.py
View File

@ -14,7 +14,6 @@ from scrapy.utils.python import (
MutableAsyncChain,
MutableChain,
binary_is_text,
equal_attributes,
get_func_args,
memoizemethod_noargs,
to_bytes,
@ -150,56 +149,6 @@ def test_binaryistext(value: bytes, expected: bool) -> None:
assert binary_is_text(value) is expected
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_equal_attributes():
class Obj:
pass
a = Obj()
b = Obj()
# no attributes given return False
assert not equal_attributes(a, b, [])
# nonexistent attributes
assert not equal_attributes(a, b, ["x", "y"])
a.x = 1
b.x = 1
# equal attribute
assert equal_attributes(a, b, ["x"])
b.y = 2
# obj1 has no attribute y
assert not equal_attributes(a, b, ["x", "y"])
a.y = 2
# equal attributes
assert equal_attributes(a, b, ["x", "y"])
a.y = 1
# different attributes
assert not equal_attributes(a, b, ["x", "y"])
# test callable
a.meta = {}
b.meta = {}
assert equal_attributes(a, b, ["meta"])
# compare ['meta']['a']
a.meta["z"] = 1
b.meta["z"] = 1
get_z = operator.itemgetter("z")
get_meta = operator.attrgetter("meta")
def compare_z(obj):
return get_z(get_meta(obj))
assert equal_attributes(a, b, [compare_z, "x"])
# fail z equality
a.meta["z"] = 2
assert not equal_attributes(a, b, [compare_z, "x"])
def test_get_func_args():
def f1(a, b, c):
pass

75
tests/test_utils_request.py
View File

@ -1,32 +1,22 @@
from __future__ import annotations
import json
import warnings
from hashlib import sha1
from weakref import WeakKeyDictionary
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.utils.python import to_bytes
from scrapy.utils.request import (
_fingerprint_cache,
fingerprint,
request_authenticate,
request_httprepr,
request_to_curl,
)
from scrapy.utils.test import get_crawler
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_request_authenticate():
r = Request("http://www.example.com")
request_authenticate(r, "someuser", "somepass")
assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M="
@pytest.mark.parametrize(
("r", "expected"),
[
@ -237,25 +227,13 @@ class TestFingerprint:
class TestRequestFingerprinter:
def test_default_implementation(self):
def test_fingerprint(self):
crawler = get_crawler()
request = Request("https://example.com")
assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
request
)
def test_deprecated_implementation(self):
settings = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
with warnings.catch_warnings(record=True) as logged_warnings:
crawler = get_crawler(settings_dict=settings)
request = Request("https://example.com")
assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
request
)
assert logged_warnings
class TestCustomRequestFingerprinter:
def test_include_headers(self):
@ -343,57 +321,6 @@ class TestCustomRequestFingerprinter:
fingerprint = crawler.request_fingerprinter.fingerprint(request)
assert fingerprint == settings["FINGERPRINT"]
def test_from_settings(self):
class RequestFingerprinter:
@classmethod
def from_settings(cls, settings):
return cls(settings)
def __init__(self, settings):
self._fingerprint = settings["FINGERPRINT"]
def fingerprint(self, request):
return self._fingerprint
settings = {
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
"FINGERPRINT": b"fingerprint",
}
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
crawler = get_crawler(settings_dict=settings)
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
assert fingerprint == settings["FINGERPRINT"]
def test_from_crawler_and_settings(self):
class RequestFingerprinter:
# This method is ignored due to the presence of from_crawler
@classmethod
def from_settings(cls, settings):
return cls(settings)
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def __init__(self, crawler):
self._fingerprint = crawler.settings["FINGERPRINT"]
def fingerprint(self, request):
return self._fingerprint
settings = {
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
"FINGERPRINT": b"fingerprint",
}
crawler = get_crawler(settings_dict=settings)
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
assert fingerprint == settings["FINGERPRINT"]
class TestRequestToCurl:
def _test_request(self, request_object, expected_curl_command):