From 61b4befc60cf7fb3f421460e168a031d7e4f1fa9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 27 Oct 2025 17:14:38 +0500 Subject: [PATCH] Remove code deprecated in 2.12.0 (#7126) * Remove smaller things. * Remove from_settings(). * Remove old MediaPipeline init code. * Cleanup. * Typo. --- docs/conf.py | 2 +- docs/topics/settings.rst | 4 +- scrapy/__init__.py | 17 ------ scrapy/core/downloader/__init__.py | 8 --- scrapy/core/downloader/contextfactory.py | 29 +-------- scrapy/dupefilters.py | 41 ++----------- scrapy/extensions/feedexport.py | 23 +------ scrapy/mail.py | 17 +----- scrapy/middleware.py | 34 +---------- scrapy/pipelines/files.py | 71 +++------------------ scrapy/pipelines/images.py | 46 +++----------- scrapy/pipelines/media.py | 75 +++-------------------- scrapy/settings/default_settings.py | 2 - scrapy/utils/defer.py | 25 -------- scrapy/utils/misc.py | 55 +---------------- scrapy/utils/python.py | 68 --------------------- scrapy/utils/request.py | 32 ---------- scrapy/utils/serialize.py | 12 ---- scrapy/utils/test.py | 16 +---- tests/test_pipeline_files.py | 31 ---------- tests/test_pipeline_media.py | 78 ------------------------ tests/test_scrapy__getattr__.py | 14 ----- tests/test_utils_misc/__init__.py | 76 +---------------------- tests/test_utils_python.py | 51 ---------------- tests/test_utils_request.py | 75 +---------------------- 25 files changed, 48 insertions(+), 854 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 5c5926dbe..9b1db9bb7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -126,7 +126,7 @@ coverage_ignore_pyobjects = [ # The interface methods of duplicate request filtering classes are already # covered in the interface documentation part of the DUPEFILTER_CLASS # setting documentation. - r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_settings|request_seen|open|close|log)$", + r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_crawler|request_seen|open|close|log)$", # Private exception used by the command-line interface implementation. r"^scrapy\.exceptions\.UsageError", # Methods of BaseItemExporter subclasses are only documented in diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5e56de05c..a92070918 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1131,9 +1131,9 @@ interface:: class MyDupeFilter: @classmethod - def from_settings(cls, settings): + def from_crawler(cls, crawler): """Returns an instance of this duplicate request filtering class - based on the current crawl settings.""" + based on the current Crawler instance.""" return cls() def request_seen(self, request): diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 280a0f659..c5b6483e3 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -29,23 +29,6 @@ __version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii"). version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split(".")) -def __getattr__(name: str): - if name == "twisted_version": - import warnings # noqa: PLC0415 # pylint: disable=reimported - - from twisted import version as _txv # noqa: PLC0415 - - from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415 - - warnings.warn( - "The scrapy.twisted_version attribute is deprecated, use twisted.version instead", - ScrapyDeprecationWarning, - ) - return _txv.major, _txv.minor, _txv.micro - - raise AttributeError - - # Ignore noisy twisted deprecation warnings warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted") diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 76cd376e1..77b17287d 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -182,14 +182,6 @@ class Downloader: return key - def _get_slot_key(self, request: Request, spider: Spider | None) -> str: - warnings.warn( - "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return self.get_slot_key(request) - # passed as download_func into self.middleware.download() in self.fetch() @inlineCallbacks def _enqueue_request( diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index d1ba6208a..5f7c9a8c3 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -71,21 +71,6 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): stacklevel=2, ) - @classmethod - def from_settings( - cls, - settings: BaseSettings, - method: int = SSL.SSLv23_METHOD, - *args: Any, - **kwargs: Any, - ) -> Self: - warnings.warn( - f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return cls._from_settings(settings, method, *args, **kwargs) - @classmethod def from_crawler( cls, @@ -94,20 +79,10 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): *args: Any, **kwargs: Any, ) -> Self: - return cls._from_settings(crawler.settings, method, *args, **kwargs) - - @classmethod - def _from_settings( - cls, - settings: BaseSettings, - method: int = SSL.SSLv23_METHOD, - *args: Any, - **kwargs: Any, - ) -> Self: - tls_verbose_logging: bool = settings.getbool( + tls_verbose_logging: bool = crawler.settings.getbool( "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" ) - tls_ciphers: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + tls_ciphers: str | None = crawler.settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] return cls( # type: ignore[misc] method=method, tls_verbose_logging=tls_verbose_logging, diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 520ec3380..2e264ade3 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -import warnings from pathlib import Path from typing import TYPE_CHECKING from warnings import warn @@ -22,7 +21,6 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.http.request import Request - from scrapy.settings import BaseSettings from scrapy.spiders import Spider @@ -30,15 +28,6 @@ class BaseDupeFilter: """Dummy duplicate request filtering class (:setting:`DUPEFILTER_CLASS`) that does not filter out any request.""" - @classmethod - def from_settings(cls, settings: BaseSettings) -> Self: - warnings.warn( - f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return cls() - @classmethod def from_crawler(cls, crawler: Crawler) -> Self: return cls() @@ -88,38 +77,16 @@ class RFPDupeFilter(BaseDupeFilter): self.file.seek(0) self.fingerprints.update(x.rstrip() for x in self.file) - @classmethod - def from_settings( - cls, - settings: BaseSettings, - *, - fingerprinter: RequestFingerprinterProtocol | None = None, - ) -> Self: - warnings.warn( - f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return cls._from_settings(settings, fingerprinter=fingerprinter) - @classmethod def from_crawler(cls, crawler: Crawler) -> Self: assert crawler.request_fingerprinter - return cls._from_settings( - crawler.settings, + debug = crawler.settings.getbool("DUPEFILTER_DEBUG") + return cls( + job_dir(crawler.settings), + debug, fingerprinter=crawler.request_fingerprinter, ) - @classmethod - def _from_settings( - cls, - settings: BaseSettings, - *, - fingerprinter: RequestFingerprinterProtocol | None = None, - ) -> Self: - debug = settings.getbool("DUPEFILTER_DEBUG") - return cls(job_dir(settings), debug, fingerprinter=fingerprinter) - def request_seen(self, request: Request) -> bool: fp = self.request_fingerprint(request) if fp in self.fingerprints: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index a850b330e..adb6fdc33 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -16,7 +16,7 @@ from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, TypeVar, cast +from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, cast from urllib.parse import unquote, urlparse from twisted.internet.defer import Deferred, DeferredList, maybeDeferred @@ -35,8 +35,6 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: - from collections.abc import Iterable - from _typeshed import OpenBinaryMode from twisted.python.failure import Failure @@ -54,25 +52,6 @@ UriParamsCallableT: TypeAlias = Callable[ [dict[str, Any], Spider], dict[str, Any] | None ] -_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") - - -def build_storage( - builder: Callable[..., _StorageT], - uri: str, - *args: Any, - feed_options: dict[str, Any] | None = None, - preargs: Iterable[Any] = (), - **kwargs: Any, -) -> _StorageT: - warnings.warn( - "scrapy.extensions.feedexport.build_storage() is deprecated, call the builder directly.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - kwargs["feed_options"] = feed_options - return builder(*preargs, uri, *args, **kwargs) - class ItemFilter: """ diff --git a/scrapy/mail.py b/scrapy/mail.py index 88612daa9..84209effa 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -7,7 +7,6 @@ See documentation in docs/topics/email.rst from __future__ import annotations import logging -import warnings from email import encoders as Encoders from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart @@ -20,7 +19,6 @@ from typing import IO, TYPE_CHECKING, Any from twisted.internet import ssl from twisted.internet.defer import Deferred -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes @@ -35,7 +33,6 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings logger = logging.getLogger(__name__) @@ -73,21 +70,9 @@ class MailSender: self.mailfrom: str = mailfrom self.debug: bool = debug - @classmethod - def from_settings(cls, settings: BaseSettings) -> Self: - warnings.warn( - f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return cls._from_settings(settings) - @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - return cls._from_settings(crawler.settings) - - @classmethod - def _from_settings(cls, settings: BaseSettings) -> Self: + settings = crawler.settings return cls( smtphost=settings["MAIL_HOST"], mailfrom=settings["MAIL_FROM"], diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 32fae7cc0..21a2ebfa3 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -23,7 +23,7 @@ if TYPE_CHECKING: from scrapy import Spider from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings, Settings + from scrapy.settings import Settings logger = logging.getLogger(__name__) @@ -100,43 +100,15 @@ class MiddlewareManager(ABC): def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: raise NotImplementedError - @staticmethod - def _build_from_settings(objcls: type[_T], settings: BaseSettings) -> _T: - if hasattr(objcls, "from_settings"): - instance = objcls.from_settings(settings) # type: ignore[attr-defined] - method_name = "from_settings" - else: - instance = objcls() - method_name = "__new__" - if instance is None: - raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return cast("_T", instance) - - @classmethod - def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self: - warnings.warn( - f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return cls._from_settings(settings, crawler) - @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - return cls._from_settings(crawler.settings, crawler) - - @classmethod - def _from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self: - mwlist = cls._get_mwlist_from_settings(settings) + mwlist = cls._get_mwlist_from_settings(crawler.settings) middlewares = [] enabled = [] for clspath in mwlist: try: mwcls = load_object(clspath) - if crawler is not None: - mw = build_from_crawler(mwcls, crawler) - else: - mw = MiddlewareManager._build_from_settings(mwcls, settings) + mw = build_from_crawler(mwcls, crawler) middlewares.append(mw) enabled.append(clspath) except NotConfigured as e: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 38f8537d6..25af0aa72 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -12,7 +12,6 @@ import hashlib import logging import mimetypes import time -import warnings from collections import defaultdict from contextlib import suppress from ftplib import FTP @@ -25,17 +24,15 @@ from itemadapter import ItemAdapter from twisted.internet.defer import Deferred, maybeDeferred from twisted.internet.threads import deferToThread -from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline -from scrapy.settings import BaseSettings, Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict -from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.python import get_func_args, global_object_name, to_bytes +from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str if TYPE_CHECKING: @@ -49,6 +46,7 @@ if TYPE_CHECKING: from scrapy import Spider from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings logger = logging.getLogger(__name__) @@ -161,7 +159,7 @@ class S3FilesStore: AWS_USE_SSL = None AWS_VERIFY = None - POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings + POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler() HEADERS = { "Cache-Control": "max-age=172800", } @@ -280,7 +278,7 @@ class GCSFilesStore: CACHE_CONTROL = "max-age=172800" # The bucket's default object ACL will be applied to the object. - # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. + # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_crawler(). POLICY = None def __init__(self, uri: str): @@ -446,9 +444,8 @@ class FilesPipeline(MediaPipeline): self, store_uri: str | PathLike[str], download_func: Callable[[Request, Spider], Response] | None = None, - settings: Settings | dict[str, Any] | None = None, *, - crawler: Crawler | None = None, + crawler: Crawler, ): if not (store_uri and (store_uri := _to_string(store_uri))): from scrapy.pipelines.images import ImagesPipeline # noqa: PLC0415 @@ -461,18 +458,7 @@ class FilesPipeline(MediaPipeline): f"to enable {self.__class__.__name__}." ) - if crawler is not None: - if settings is not None: - warnings.warn( - f"FilesPipeline.__init__() was called with a crawler instance and a settings instance" - f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored" - f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - settings = crawler.settings - elif isinstance(settings, dict) or settings is None: - settings = Settings(settings) + settings = crawler.settings cls_name = "FilesPipeline" self.store: FilesStoreProtocol = self._get_store(store_uri) resolve = functools.partial( @@ -490,51 +476,14 @@ class FilesPipeline(MediaPipeline): resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD ) - super().__init__( - download_func=download_func, - settings=settings if not crawler else None, - crawler=crawler, - ) - - @classmethod - def from_settings(cls, settings: Settings) -> Self: - warnings.warn( - f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return cls._from_settings(settings, None) + super().__init__(download_func=download_func, crawler=crawler) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - if method_is_overridden(cls, FilesPipeline, "from_settings"): - warnings.warn( - f"{global_object_name(cls)} overrides FilesPipeline.from_settings()." - f" This method is deprecated and won't be called in future Scrapy versions," - f" please update your code so that it overrides from_crawler() instead.", - category=ScrapyDeprecationWarning, - ) - o = cls.from_settings(crawler.settings) - o._finish_init(crawler) - return o - return cls._from_settings(crawler.settings, crawler) - - @classmethod - def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self: + settings = crawler.settings cls._update_stores(settings) store_uri = settings["FILES_STORE"] - if "crawler" in get_func_args(cls.__init__): - o = cls(store_uri, crawler=crawler) - else: - o = cls(store_uri, settings=settings) - if crawler: - o._finish_init(crawler) - warnings.warn( - f"{global_object_name(cls)}.__init__() doesn't take a crawler argument." - " This is deprecated and the argument will be required in future Scrapy versions.", - category=ScrapyDeprecationWarning, - ) - return o + return cls(store_uri, crawler=crawler) @classmethod def _update_stores(cls, settings: BaseSettings) -> None: diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 5ac211dfb..687d29fa5 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -8,19 +8,17 @@ from __future__ import annotations import functools import hashlib -import warnings from contextlib import suppress from io import BytesIO from typing import TYPE_CHECKING, Any from itemadapter import ItemAdapter -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum -from scrapy.settings import Settings -from scrapy.utils.python import get_func_args, global_object_name, to_bytes +from scrapy.utils.python import to_bytes if TYPE_CHECKING: from collections.abc import Callable, Iterable @@ -58,9 +56,8 @@ class ImagesPipeline(FilesPipeline): self, store_uri: str | PathLike[str], download_func: Callable[[Request, Spider], Response] | None = None, - settings: Settings | dict[str, Any] | None = None, *, - crawler: Crawler | None = None, + crawler: Crawler, ): try: from PIL import Image, ImageOps # noqa: PLC0415 @@ -72,26 +69,9 @@ class ImagesPipeline(FilesPipeline): "ImagesPipeline requires installing Pillow 8.3.2 or later" ) - super().__init__( - store_uri, - settings=settings if not crawler else None, - download_func=download_func, - crawler=crawler, - ) - - if crawler is not None: - if settings is not None: - warnings.warn( - f"ImagesPipeline.__init__() was called with a crawler instance and a settings instance" - f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored" - f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - settings = crawler.settings - elif isinstance(settings, dict) or settings is None: - settings = Settings(settings) + super().__init__(store_uri, download_func=download_func, crawler=crawler) + settings = crawler.settings resolve = functools.partial( self._key_for_pipe, base_class_name="ImagesPipeline", @@ -121,21 +101,11 @@ class ImagesPipeline(FilesPipeline): ) @classmethod - def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: + settings = crawler.settings cls._update_stores(settings) store_uri = settings["IMAGES_STORE"] - if "crawler" in get_func_args(cls.__init__): - o = cls(store_uri, crawler=crawler) - else: - o = cls(store_uri, settings=settings) - if crawler: - o._finish_init(crawler) - warnings.warn( - f"{global_object_name(cls)}.__init__() doesn't take a crawler argument." - " This is deprecated and the argument will be required in future Scrapy versions.", - category=ScrapyDeprecationWarning, - ) - return o + return cls(store_uri, crawler=crawler) def file_downloaded( self, diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index f8fc3687e..438e5e7d6 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -19,14 +19,13 @@ from twisted.python.versions import Version from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http.request import NO_CALLBACK, Request -from scrapy.settings import Settings from scrapy.utils.asyncio import call_later from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.defer import _DEFER_DELAY, _defer_sleep, deferred_from_coro from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import get_func_args, global_object_name +from scrapy.utils.python import global_object_name if TYPE_CHECKING: from collections.abc import Callable, Generator @@ -37,6 +36,7 @@ if TYPE_CHECKING: from scrapy import Spider from scrapy.crawler import Crawler from scrapy.http import Response + from scrapy.settings import Settings from scrapy.utils.request import RequestFingerprinterProtocol @@ -55,10 +55,6 @@ logger = logging.getLogger(__name__) class MediaPipeline(ABC): - crawler: Crawler - _fingerprinter: RequestFingerprinterProtocol - _modern_init = False - LOG_FAILED_RESULTS: bool = True class SpiderInfo: @@ -73,24 +69,17 @@ class MediaPipeline(ABC): def __init__( self, download_func: Callable[[Request, Spider], Response] | None = None, - settings: Settings | dict[str, Any] | None = None, *, - crawler: Crawler | None = None, + crawler: Crawler, ): + self.crawler: Crawler = crawler + assert crawler.request_fingerprinter + self._fingerprinter: RequestFingerprinterProtocol = ( + crawler.request_fingerprinter + ) self.download_func = download_func - if crawler is not None: - if settings is not None: - warnings.warn( - f"MediaPipeline.__init__() was called with a crawler instance and a settings instance" - f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored" - f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - settings = crawler.settings - elif isinstance(settings, dict) or settings is None: - settings = Settings(settings) + settings = crawler.settings resolve = functools.partial( self._key_for_pipe, base_class_name="MediaPipeline", settings=settings ) @@ -99,27 +88,6 @@ class MediaPipeline(ABC): ) self._handle_statuses(self.allow_redirects) - if crawler: - self._finish_init(crawler) - self._modern_init = True - else: - warnings.warn( - f"MediaPipeline.__init__() was called without the crawler argument" - f" when creating {global_object_name(self.__class__)}." - f" This is deprecated and the argument will be required in future Scrapy versions.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - - def _finish_init(self, crawler: Crawler) -> None: - # This was done in from_crawler() before 2.12, now it's done in __init__() - # if the crawler was passed to it and may be needed to be called in other - # deprecated code paths explicitly too. After the crawler argument of __init__() - # becomes mandatory this should be inlined there. - self.crawler = crawler - assert crawler.request_fingerprinter - self._fingerprinter = crawler.request_fingerprinter - def _handle_statuses(self, allow_redirects: bool) -> None: self.handle_httpstatus_list = None if allow_redirects: @@ -143,30 +111,7 @@ class MediaPipeline(ABC): @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - pipe: Self - if hasattr(cls, "from_settings"): - pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined] - warnings.warn( - f"{global_object_name(cls)} has from_settings() and either doesn't have" - " from_crawler() or calls MediaPipeline.from_crawler() from it," - " so from_settings() was used to create the instance of it." - " This is deprecated and calling from_settings() will be removed" - " in a future Scrapy version. Please move the initialization code into" - " from_crawler() or __init__().", - category=ScrapyDeprecationWarning, - ) - elif "crawler" in get_func_args(cls.__init__): - pipe = cls(crawler=crawler) - else: - pipe = cls() - warnings.warn( - f"{global_object_name(cls)}.__init__() doesn't take a crawler argument." - " This is deprecated and the argument will be required in future Scrapy versions.", - category=ScrapyDeprecationWarning, - ) - if not pipe._modern_init: - pipe._finish_init(crawler) - return pipe + return cls(crawler=crawler) @_warn_spider_arg def open_spider(self, spider: Spider | None = None) -> None: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 543e5c043..e66f78c47 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -152,7 +152,6 @@ __all__ = [ "REFERER_ENABLED", "REFERRER_POLICY", "REQUEST_FINGERPRINTER_CLASS", - "REQUEST_FINGERPRINTER_IMPLEMENTATION", "RETRY_ENABLED", "RETRY_EXCEPTIONS", "RETRY_HTTP_CODES", @@ -451,7 +450,6 @@ REFERER_ENABLED = True REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" -REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL" RETRY_ENABLED = True RETRY_EXCEPTIONS = [ diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 39cd6e540..8ca3bc0bd 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -317,31 +317,6 @@ def process_chain( return d -def process_chain_both( - callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], - errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]], - input: Any, # noqa: A002 - *a: _P.args, - **kw: _P.kwargs, -) -> Deferred: - """Return a Deferred built by chaining the given callbacks and errbacks""" - warnings.warn( - "process_chain_both() is deprecated and will be removed in a future" - " Scrapy version.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - d: Deferred = Deferred() - for cb, eb in zip(callbacks, errbacks, strict=False): - d.addCallback(cb, *a, **kw) - d.addErrback(eb, *a, **kw) - if isinstance(input, failure.Failure): - d.errback(input) - else: - d.callback(input) - return d - - def process_parallel( callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]], input: _T, # noqa: A002 diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 1acb06757..7994a2471 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -129,52 +129,10 @@ def rel_has_nofollow(rel: str | None) -> bool: return rel is not None and "nofollow" in rel.replace(",", " ").split() -def create_instance(objcls, settings, crawler, *args, **kwargs): - """Construct a class instance using its ``from_crawler`` or - ``from_settings`` constructors, if available. - - At least one of ``settings`` and ``crawler`` needs to be different from - ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used. - If ``crawler`` is ``None``, only the ``from_settings`` constructor will be - tried. - - ``*args`` and ``**kwargs`` are forwarded to the constructors. - - Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``. - - .. versionchanged:: 2.2 - Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an - extension has not been implemented correctly). - """ - warnings.warn( - "The create_instance() function is deprecated. " - "Please use build_from_crawler() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - - if settings is None: - if crawler is None: - raise ValueError("Specify at least one of settings and crawler.") - settings = crawler.settings - if crawler and hasattr(objcls, "from_crawler"): - instance = objcls.from_crawler(crawler, *args, **kwargs) - method_name = "from_crawler" - elif hasattr(objcls, "from_settings"): - instance = objcls.from_settings(settings, *args, **kwargs) - method_name = "from_settings" - else: - instance = objcls(*args, **kwargs) - method_name = "__new__" - if instance is None: - raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return instance - - def build_from_crawler( objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any ) -> T: - """Construct a class instance using its ``from_crawler`` or ``from_settings`` constructor. + """Construct a class instance using its ``from_crawler()`` or ``__init__()`` constructor. .. versionadded:: 2.12 @@ -185,17 +143,6 @@ def build_from_crawler( if hasattr(objcls, "from_crawler"): instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined] method_name = "from_crawler" - elif hasattr(objcls, "from_settings"): - warnings.warn( - f"{objcls.__qualname__} has from_settings() but not from_crawler()." - " This is deprecated and calling from_settings() will be removed in a future" - " Scrapy version. You can implement a simple from_crawler() that calls" - " from_settings() with crawler.settings.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - instance = objcls.from_settings(crawler.settings, *args, **kwargs) # type: ignore[attr-defined] - method_name = "from_settings" else: instance = objcls(*args, **kwargs) method_name = "__new__" diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index d9a5007cc..ca4b4bf28 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,14 +8,12 @@ import gc import inspect import re import sys -import warnings import weakref from collections.abc import AsyncIterator, Iterable, Mapping from functools import partial, wraps from itertools import chain from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: @@ -32,47 +30,6 @@ _VT = TypeVar("_VT") _P = ParamSpec("_P") -def flatten(x: Iterable[Any]) -> list[Any]: - """flatten(sequence) -> list - - Returns a single, flat list which contains all elements retrieved - from the sequence and all recursively contained sub-sequences - (iterables). - - Examples: - >>> [1, 2, [3,4], (5,6)] - [1, 2, [3, 4], (5, 6)] - >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)]) - [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10] - >>> flatten(["foo", "bar"]) - ['foo', 'bar'] - >>> flatten(["foo", ["baz", 42], "bar"]) - ['foo', 'baz', 42, 'bar'] - """ - warnings.warn( - "The flatten function is deprecated and will be removed in a future version of Scrapy.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return list(iflatten(x)) - - -def iflatten(x: Iterable[Any]) -> Iterable[Any]: - """iflatten(sequence) -> iterator - - Similar to ``.flatten()``, but returns iterator instead""" - warnings.warn( - "The iflatten function is deprecated and will be removed in a future version of Scrapy.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - for el in x: - if is_listlike(el): - yield from iflatten(el) - else: - yield el - - def is_listlike(x: Any) -> bool: """ >>> is_listlike("foo") @@ -289,31 +246,6 @@ def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]: return args, kwargs -def equal_attributes( - obj1: Any, obj2: Any, attributes: list[str | Callable[[Any], Any]] | None -) -> bool: - """Compare two objects attributes""" - warnings.warn( - "The equal_attributes function is deprecated and will be removed in a future version of Scrapy.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - # not attributes given return False by default - if not attributes: - return False - - temp1, temp2 = object(), object() - for attr in attributes: - # support callables like itemgetter - if callable(attr): - if attr(obj1) != attr(obj2): - return False - elif getattr(obj1, attr, temp1) != getattr(obj2, attr, temp2): - return False - # all attributes equal - return True - - @overload def without_none_values(iterable: Mapping[_KT, _VT]) -> dict[_KT, _VT]: ... diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 9c1161968..616ab661f 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -7,16 +7,13 @@ from __future__ import annotations import hashlib import json -import warnings from typing import TYPE_CHECKING, Any, Protocol from urllib.parse import urlunparse from weakref import WeakKeyDictionary -from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url from scrapy import Request, Spider -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode @@ -120,41 +117,12 @@ class RequestFingerprinter: return cls(crawler) def __init__(self, crawler: Crawler | None = None): - if crawler: - implementation = crawler.settings.get( - "REQUEST_FINGERPRINTER_IMPLEMENTATION" - ) - else: - implementation = "SENTINEL" - - if implementation != "SENTINEL": - message = ( - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n" - "It will be removed in a future version of Scrapy." - ) - warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) self._fingerprint = fingerprint def fingerprint(self, request: Request) -> bytes: return self._fingerprint(request) -def request_authenticate( - request: Request, - username: str, - password: str, -) -> None: - """Authenticate the given request (in place) using the HTTP basic access - authentication mechanism (RFC 2617) and the given username and password - """ - warnings.warn( - "The request_authenticate function is deprecated and will be removed in a future version of Scrapy.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - request.headers["Authorization"] = basic_auth_header(username, password) - - def request_httprepr(request: Request) -> bytes: """Return the raw HTTP representation (as bytes) of the given request. This is provided only for reference since it's not the actual stream of diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index bcfae0c00..5d06bbe30 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -1,13 +1,11 @@ import datetime import decimal import json -import warnings from typing import Any from itemadapter import ItemAdapter, is_item from twisted.internet import defer -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response @@ -35,13 +33,3 @@ class ScrapyJSONEncoder(json.JSONEncoder): if is_item(o): return ItemAdapter(o).asdict() return super().default(o) - - -class ScrapyJSONDecoder(json.JSONDecoder): - def __init__(self, *args, **kwargs): - warnings.warn( - "The ScrapyJSONDecoder class is deprecated and will be removed in a future version of Scrapy.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - super().__init__(*args, **kwargs) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4c0764396..4e0356350 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -12,7 +12,7 @@ from importlib import import_module from pathlib import Path from posixpath import split from typing import TYPE_CHECKING, Any, TypeVar, cast -from unittest import TestCase, mock +from unittest import mock from twisted.trial.unittest import SkipTest from twisted.web.client import Agent @@ -166,20 +166,6 @@ def get_testenv() -> dict[str, str]: return env -def assert_samelines( - testcase: TestCase, text1: str, text2: str, msg: str | None = None -) -> None: - """Asserts text1 and text2 have the same lines, ignoring differences in - line endings between platforms - """ - warnings.warn( - "The assert_samelines function is deprecated and will be removed in a future version of Scrapy.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) # noqa: PT009 - - def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: q: asyncio.Queue[_T] = asyncio.Queue() getter = q.get() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 99d5cbb6d..966cb320e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -756,37 +756,6 @@ class TestBuildFromCrawler: assert len(w) == 0 assert pipe.store - def test_has_old_init(self): - class Pipeline(FilesPipeline): - def __init__(self, store_uri, download_func=None, settings=None): - super().__init__(store_uri, download_func, settings) - self._init_called = True - - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 2 - assert pipe._init_called - - def test_has_from_settings(self): - class Pipeline(FilesPipeline): - _from_settings_called = False - - @classmethod - def from_settings(cls, settings): - o = super().from_settings(settings) - o._from_settings_called = True - return o - - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 3 - assert pipe.store - assert pipe._from_settings_called - def test_has_from_crawler_and_init(self): class Pipeline(FilesPipeline): _from_crawler_called = False diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index cc1d73349..9f174cc5f 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -418,84 +418,6 @@ class TestBuildFromCrawler: assert pipe._fingerprinter assert len(w) == 0 - def test_has_old_init(self): - class Pipeline(UserDefinedPipeline): - def __init__(self): - super().__init__() - self._init_called = True - - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 2 - assert pipe._init_called - - def test_has_from_settings(self): - class Pipeline(UserDefinedPipeline): - _from_settings_called = False - - @classmethod - def from_settings(cls, settings): - o = cls() - o._from_settings_called = True - return o - - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 2 - assert pipe._from_settings_called - - def test_has_from_settings_and_from_crawler(self): - class Pipeline(UserDefinedPipeline): - _from_settings_called = False - _from_crawler_called = False - - @classmethod - def from_settings(cls, settings): - o = cls() - o._from_settings_called = True - return o - - @classmethod - def from_crawler(cls, crawler): - o = super().from_crawler(crawler) - o._from_crawler_called = True - return o - - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 2 - assert pipe._from_settings_called - assert pipe._from_crawler_called - - def test_has_from_settings_and_init(self): - class Pipeline(UserDefinedPipeline): - _from_settings_called = False - - def __init__(self, store_uri, settings): - super().__init__() - self._init_called = True - - @classmethod - def from_settings(cls, settings): - store_uri = settings["FILES_STORE"] - o = cls(store_uri, settings=settings) - o._from_settings_called = True - return o - - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 2 - assert pipe._from_settings_called - assert pipe._init_called - def test_has_from_crawler_and_init(self): class Pipeline(UserDefinedPipeline): _from_crawler_called = False diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py index fa6d11123..4c365859b 100644 --- a/tests/test_scrapy__getattr__.py +++ b/tests/test_scrapy__getattr__.py @@ -1,20 +1,6 @@ import warnings -def test_deprecated_twisted_version(): - with warnings.catch_warnings(record=True) as warns: - from scrapy import ( # noqa: PLC0415 # pylint: disable=no-name-in-module - twisted_version, - ) - - assert twisted_version is not None - assert isinstance(twisted_version, tuple) - assert ( - "The scrapy.twisted_version attribute is deprecated, use twisted.version instead" - in warns[0].message.args - ) - - def test_deprecated_concurrent_requests_per_ip_attribute(): with warnings.catch_warnings(record=True) as warns: from scrapy.settings.default_settings import ( # noqa: PLC0415 diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index b330819d9..610fc0ffe 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -9,7 +9,6 @@ from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, build_from_crawler, - create_instance, load_object, rel_has_nofollow, set_environ, @@ -97,98 +96,29 @@ class TestUtilsMisc: assert list(arg_to_iter({"a": 1})) == [{"a": 1}] assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_create_instance(self): - settings = mock.MagicMock() - crawler = mock.MagicMock(spec_set=["settings"]) - args = (True, 100.0) - kwargs = {"key": "val"} - - def _test_with_settings(mock, settings): - create_instance(mock, settings, None, *args, **kwargs) - if hasattr(mock, "from_crawler"): - assert mock.from_crawler.call_count == 0 - if hasattr(mock, "from_settings"): - mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - assert mock.call_count == 0 - else: - mock.assert_called_once_with(*args, **kwargs) - - def _test_with_crawler(mock, settings, crawler): - create_instance(mock, settings, crawler, *args, **kwargs) - if hasattr(mock, "from_crawler"): - mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) - if hasattr(mock, "from_settings"): - assert mock.from_settings.call_count == 0 - assert mock.call_count == 0 - elif hasattr(mock, "from_settings"): - mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - assert mock.call_count == 0 - else: - mock.assert_called_once_with(*args, **kwargs) - - # Check usage of correct constructor using four mocks: - # 1. with no alternative constructors - # 2. with from_settings() constructor - # 3. with from_crawler() constructor - # 4. with from_settings() and from_crawler() constructor - spec_sets = ( - ["__qualname__"], - ["__qualname__", "from_settings"], - ["__qualname__", "from_crawler"], - ["__qualname__", "from_settings", "from_crawler"], - ) - for specs in spec_sets: - m = mock.MagicMock(spec_set=specs) - _test_with_settings(m, settings) - m.reset_mock() - _test_with_crawler(m, settings, crawler) - - # Check adoption of crawler settings - m = mock.MagicMock(spec_set=["__qualname__", "from_settings"]) - create_instance(m, None, crawler, *args, **kwargs) - m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs) - - with pytest.raises( - ValueError, match="Specify at least one of settings and crawler" - ): - create_instance(m, None, None) - - m.from_settings.return_value = None - with pytest.raises(TypeError): - create_instance(m, settings, None) - def test_build_from_crawler(self): - settings = mock.MagicMock() crawler = mock.MagicMock(spec_set=["settings"]) args = (True, 100.0) kwargs = {"key": "val"} - def _test_with_crawler(mock, settings, crawler): + def _test_with_crawler(mock, crawler): build_from_crawler(mock, crawler, *args, **kwargs) if hasattr(mock, "from_crawler"): mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) - if hasattr(mock, "from_settings"): - assert mock.from_settings.call_count == 0 - assert mock.call_count == 0 - elif hasattr(mock, "from_settings"): - mock.from_settings.assert_called_once_with(settings, *args, **kwargs) assert mock.call_count == 0 else: mock.assert_called_once_with(*args, **kwargs) - # Check usage of correct constructor using three mocks: + # Check usage of correct constructor using 2 mocks: # 1. with no alternative constructors # 2. with from_crawler() constructor - # 3. with from_settings() and from_crawler() constructor spec_sets = ( ["__qualname__"], ["__qualname__", "from_crawler"], - ["__qualname__", "from_settings", "from_crawler"], ) for specs in spec_sets: m = mock.MagicMock(spec_set=specs) - _test_with_crawler(m, settings, crawler) + _test_with_crawler(m, crawler) m.reset_mock() # Check adoption of crawler diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index cf1c6281d..34fca06c1 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -14,7 +14,6 @@ from scrapy.utils.python import ( MutableAsyncChain, MutableChain, binary_is_text, - equal_attributes, get_func_args, memoizemethod_noargs, to_bytes, @@ -150,56 +149,6 @@ def test_binaryistext(value: bytes, expected: bool) -> None: assert binary_is_text(value) is expected -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -def test_equal_attributes(): - class Obj: - pass - - a = Obj() - b = Obj() - # no attributes given return False - assert not equal_attributes(a, b, []) - # nonexistent attributes - assert not equal_attributes(a, b, ["x", "y"]) - - a.x = 1 - b.x = 1 - # equal attribute - assert equal_attributes(a, b, ["x"]) - - b.y = 2 - # obj1 has no attribute y - assert not equal_attributes(a, b, ["x", "y"]) - - a.y = 2 - # equal attributes - assert equal_attributes(a, b, ["x", "y"]) - - a.y = 1 - # different attributes - assert not equal_attributes(a, b, ["x", "y"]) - - # test callable - a.meta = {} - b.meta = {} - assert equal_attributes(a, b, ["meta"]) - - # compare ['meta']['a'] - a.meta["z"] = 1 - b.meta["z"] = 1 - - get_z = operator.itemgetter("z") - get_meta = operator.attrgetter("meta") - - def compare_z(obj): - return get_z(get_meta(obj)) - - assert equal_attributes(a, b, [compare_z, "x"]) - # fail z equality - a.meta["z"] = 2 - assert not equal_attributes(a, b, [compare_z, "x"]) - - def test_get_func_args(): def f1(a, b, c): pass diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 9c4cb7159..b25129c87 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,32 +1,22 @@ from __future__ import annotations import json -import warnings from hashlib import sha1 from weakref import WeakKeyDictionary import pytest -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.utils.python import to_bytes from scrapy.utils.request import ( _fingerprint_cache, fingerprint, - request_authenticate, request_httprepr, request_to_curl, ) from scrapy.utils.test import get_crawler -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -def test_request_authenticate(): - r = Request("http://www.example.com") - request_authenticate(r, "someuser", "somepass") - assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M=" - - @pytest.mark.parametrize( ("r", "expected"), [ @@ -237,25 +227,13 @@ class TestFingerprint: class TestRequestFingerprinter: - def test_default_implementation(self): + def test_fingerprint(self): crawler = get_crawler() request = Request("https://example.com") assert crawler.request_fingerprinter.fingerprint(request) == fingerprint( request ) - def test_deprecated_implementation(self): - settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - } - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(settings_dict=settings) - request = Request("https://example.com") - assert crawler.request_fingerprinter.fingerprint(request) == fingerprint( - request - ) - assert logged_warnings - class TestCustomRequestFingerprinter: def test_include_headers(self): @@ -343,57 +321,6 @@ class TestCustomRequestFingerprinter: fingerprint = crawler.request_fingerprinter.fingerprint(request) assert fingerprint == settings["FINGERPRINT"] - def test_from_settings(self): - class RequestFingerprinter: - @classmethod - def from_settings(cls, settings): - return cls(settings) - - def __init__(self, settings): - self._fingerprint = settings["FINGERPRINT"] - - def fingerprint(self, request): - return self._fingerprint - - settings = { - "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, - "FINGERPRINT": b"fingerprint", - } - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - crawler = get_crawler(settings_dict=settings) - - request = Request("http://www.example.com") - fingerprint = crawler.request_fingerprinter.fingerprint(request) - assert fingerprint == settings["FINGERPRINT"] - - def test_from_crawler_and_settings(self): - class RequestFingerprinter: - # This method is ignored due to the presence of from_crawler - @classmethod - def from_settings(cls, settings): - return cls(settings) - - @classmethod - def from_crawler(cls, crawler): - return cls(crawler) - - def __init__(self, crawler): - self._fingerprint = crawler.settings["FINGERPRINT"] - - def fingerprint(self, request): - return self._fingerprint - - settings = { - "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, - "FINGERPRINT": b"fingerprint", - } - crawler = get_crawler(settings_dict=settings) - - request = Request("http://www.example.com") - fingerprint = crawler.request_fingerprinter.fingerprint(request) - assert fingerprint == settings["FINGERPRINT"] - class TestRequestToCurl: def _test_request(self, request_object, expected_curl_command):