From 394c2797f34094299dfcfd621d7315947cb8dd05 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 21 Jul 2026 16:30:21 +0500 Subject: [PATCH] Remove non-Twisted uses of zope.interface (#7731) --- docs/news.rst | 24 ++++++++++++- docs/topics/api.rst | 45 ++++-------------------- scrapy/core/downloader/contextfactory.py | 2 -- scrapy/extensions/feedexport.py | 33 ++++++++--------- scrapy/interfaces.py | 24 +++++++------ scrapy/spiderloader.py | 44 +++++++++++++---------- tests/test_crawler_runners.py | 20 ----------- tests/test_feedexport.py | 5 --- tests/test_feedexport_batch.py | 6 +--- tests/test_feedexport_storages.py | 9 ----- tests/test_spiderloader/__init__.py | 5 --- 11 files changed, 87 insertions(+), 130 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 63f0d69e4..4ebab3c6a 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,28 @@ Release notes ============= +Scrapy VERSION (unreleased) +--------------------------- + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The following runtime usage of zope.interface_ interfaces is removed: + + - :class:`~scrapy.spiderloader.SpiderLoader` and + :class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked + as implementing the ``ISpiderLoader`` interface. + + - :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the + configured spider loader implements the ``ISpiderLoader`` interface. + + - :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`, + :class:`~scrapy.extensions.feedexport.FileFeedStorage` and + :class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer + marked as implementing the ``IFeedStorage`` interface. + + (:issue:`6585`, :issue:`7731`) + .. _release-2.17.0: Scrapy 2.17.0 (2026-07-07) @@ -2643,7 +2665,7 @@ Deprecation removals (:issue:`6109`, :issue:`6116`) - A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that - does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface + does not implement the ``ISpiderLoader`` interface will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at run time. Non-compliant classes have been triggering a deprecation warning since Scrapy 1.0.0. diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 19082d9d7..7ff8d3464 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -172,46 +172,15 @@ SpiderLoader API .. module:: scrapy.spiderloader :synopsis: The spider loader -.. class:: SpiderLoader +Custom spider loaders can be employed by specifying their path in the +:setting:`SPIDER_LOADER_CLASS` project setting. They must implement +:class:`SpiderLoaderProtocol`. - This class is in charge of retrieving and handling the spider classes - defined across the project. +.. autoclass:: SpiderLoaderProtocol + :members: - Custom spider loaders can be employed by specifying their path in the - :setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement - the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an - errorless execution. - - .. method:: from_settings(settings) - - This class method is used by Scrapy to create an instance of the class. - It's called with the current project settings, and it loads the spiders - found recursively in the modules of the :setting:`SPIDER_MODULES` - setting. - - :param settings: project settings - :type settings: :class:`~scrapy.settings.Settings` instance - - .. method:: load(spider_name) - - Get the Spider class with the given name. It'll look into the previously - loaded spiders for a spider class with name ``spider_name`` and will raise - a KeyError if not found. - - :param spider_name: spider class name - :type spider_name: str - - .. method:: list() - - Get the names of the available spiders in the project. - - .. method:: find_by_request(request) - - List the spiders' names that can handle the given request. Will try to - match the request's url against the domains of the spiders. - - :param request: queried request - :type request: :class:`~scrapy.Request` instance +.. autoclass:: SpiderLoader + :members: .. autoclass:: DummySpiderLoader diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index a934cbbc7..ef948997d 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -13,7 +13,6 @@ from twisted.internet.ssl import ( from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer -from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import ( _TWISTED_VERSION_MAP, @@ -232,7 +231,6 @@ class _AcceptableProtocolsContextFactory: # all of this with _ScrapyClientContextFactory.acceptableProtocols. def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]): - verifyObject(IPolicyForHTTPS, context_factory) self._wrapped_context_factory: Any = context_factory self._acceptable_protocols: list[bytes] = acceptable_protocols diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0d9b016a3..678a29e2e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -22,7 +22,7 @@ from urllib.parse import unquote, urlparse from twisted.internet.defer import Deferred, DeferredList from w3lib.url import file_uri_to_path -from zope.interface import Interface, implementer +from zope.interface import Interface from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning @@ -115,25 +115,18 @@ class ItemFilter: return True # accept all items by default -class IFeedStorage(Interface): # type: ignore[misc] - """Interface that all Feed Storages must implement""" - +class _IFeedStorage(Interface): # type: ignore[misc] # pragma: no cover # pylint: disable=no-self-argument - def __init__(uri, *, feed_options=None): # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called - """Initialize the storage with the parameters given in the URI and the - feed-specific options (see :setting:`FEEDS`)""" + def __init__(uri, *, feed_options=None): ... # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called - def open(spider): # type: ignore[no-untyped-def] - """Open the storage for the given spider. It must return a file-like - object that will be used for the exporters""" + def open(spider): ... # type: ignore[no-untyped-def] - def store(file): # type: ignore[no-untyped-def] - """Store the given file stream""" + def store(file): ... # type: ignore[no-untyped-def] class FeedStorageProtocol(Protocol): - """Reimplementation of ``IFeedStorage`` that can be used in type hints.""" + """Protocol that all Feed Storages must follow.""" def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): """Initialize the storage with the parameters given in the URI and the @@ -147,7 +140,6 @@ class FeedStorageProtocol(Protocol): """Store the given file stream""" -@implementer(IFeedStorage) class BlockingFeedStorage(ABC): def open(self, spider: Spider) -> IO[bytes]: path = spider.crawler.settings["FEED_TEMPDIR"] @@ -164,7 +156,6 @@ class BlockingFeedStorage(ABC): raise NotImplementedError -@implementer(IFeedStorage) class StdoutFeedStorage: def __init__( self, @@ -191,7 +182,6 @@ class StdoutFeedStorage: pass -@implementer(IFeedStorage) class FileFeedStorage: def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri @@ -758,3 +748,14 @@ class FeedExporter: feed_options.get("item_filter", ItemFilter) ) return item_filter_class(feed_options) + + +def __getattr__(name: str) -> Any: # pragma: no cover + if name == "IFeedStorage": + warnings.warn( + "scrapy.extensions.feedexport.IFeedStorage is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _IFeedStorage + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index b4f1d9394..34d7bdd82 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,19 +1,23 @@ +# pragma: no file cover # pylint: disable=no-method-argument,no-self-argument +import warnings from zope.interface import Interface +from scrapy.exceptions import ScrapyDeprecationWarning + +warnings.warn( + "The scrapy.interfaces module is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, +) + class ISpiderLoader(Interface): - def from_settings(settings): - """Return an instance of the class for the given settings""" + def from_settings(settings): ... - def load(spider_name): - """Return the Spider class for the given spider name. If the spider - name is not found, it must raise a KeyError.""" + def load(spider_name): ... - def list(): - """Return a list with the names of all spiders available in the - project""" + def list(): ... - def find_by_request(request): - """Return the list of spiders names that can handle the given request""" + def find_by_request(request): ... diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 8c980fd46..244b7338f 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -5,10 +5,8 @@ import warnings from collections import defaultdict from typing import TYPE_CHECKING, Protocol, cast -from zope.interface import implementer -from zope.interface.verify import verifyClass - -from scrapy.interfaces import ISpiderLoader +# working around https://github.com/sphinx-doc/sphinx/issues/10400 +from scrapy import Request, Spider # noqa: TC001 from scrapy.utils.misc import load_object, walk_modules_iter from scrapy.utils.spider import iter_spider_classes @@ -18,7 +16,6 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self - from scrapy import Request, Spider from scrapy.settings import BaseSettings @@ -26,28 +23,31 @@ def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) - verifyClass(ISpiderLoader, loader_cls) return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())) class SpiderLoaderProtocol(Protocol): + """Protocol for spider loader implementations. + + See :setting:`SPIDER_LOADER_CLASS`. + """ + @classmethod def from_settings(cls, settings: BaseSettings) -> Self: - """Return an instance of the class for the given settings""" + """Return an instance of the class for the given settings.""" def load(self, spider_name: str) -> type[Spider]: - """Return the Spider class for the given spider name. If the spider - name is not found, it must raise a KeyError.""" + """Return the spider class for the given spider name. If the spider + name is not found, it must raise a :exc:`KeyError`.""" def list(self) -> list[str]: """Return a list with the names of all spiders available in the - project""" + project.""" def find_by_request(self, request: Request) -> __builtins__.list[str]: - """Return the list of spiders names that can handle the given request""" + """Return the list of spiders names that can handle the given request.""" -@implementer(ISpiderLoader) class SpiderLoader: """ SpiderLoader is a class which locates and loads spiders @@ -106,12 +106,18 @@ class SpiderLoader: @classmethod def from_settings(cls, settings: BaseSettings) -> Self: + """Create an instance of the class. + + It's called with the current project settings, and it loads the spiders + found recursively in the modules of the :setting:`SPIDER_MODULES` + setting. + """ return cls(settings) def load(self, spider_name: str) -> type[Spider]: - """ - Return the Spider class for the given spider name. If the spider - name is not found, raise a KeyError. + """Return the spider class for the given spider name. + + If the spider name is not found, raise a :exc:`KeyError`. """ try: return self._spiders[spider_name] @@ -121,19 +127,19 @@ class SpiderLoader: def find_by_request(self, request: Request) -> list[str]: """ Return the list of spider names that can handle the given request. + + It will try to match the request's url against the domains of + the spiders. """ return [ name for name, cls in self._spiders.items() if cls.handles_request(request) ] def list(self) -> list[str]: - """ - Return a list with the names of all spiders available in the project. - """ + """Return a list with the names of all spiders available in the project.""" return list(self._spiders.keys()) -@implementer(ISpiderLoader) class DummySpiderLoader: """A dummy spider loader that does not load any spiders.""" diff --git a/tests/test_crawler_runners.py b/tests/test_crawler_runners.py index 2f3649b43..9e35a38e2 100644 --- a/tests/test_crawler_runners.py +++ b/tests/test_crawler_runners.py @@ -4,7 +4,6 @@ import logging from typing import TYPE_CHECKING import pytest -from zope.interface.exceptions import MultipleInvalid from scrapy.crawler import ( AsyncCrawlerProcess, @@ -14,7 +13,6 @@ from scrapy.crawler import ( CrawlerRunner, CrawlerRunnerBase, ) -from scrapy.settings import Settings from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_reactor_settings @@ -33,15 +31,6 @@ class SpiderLoaderWithWrongInterface: class TestCrawlerRunner: - def test_spider_manager_verify_interface(self) -> None: - settings = Settings( - { - "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, - } - ) - with pytest.raises(MultipleInvalid): - CrawlerRunner(settings) - def test_crawler_runner_accepts_dict(self) -> None: runner = CrawlerRunner({"foo": "bar"}) assert runner.settings["foo"] == "bar" @@ -53,15 +42,6 @@ class TestCrawlerRunner: class TestAsyncCrawlerRunner: - def test_spider_manager_verify_interface(self) -> None: - settings = Settings( - { - "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, - } - ) - with pytest.raises(MultipleInvalid): - AsyncCrawlerRunner(settings) - def test_crawler_runner_accepts_dict(self) -> None: runner = AsyncCrawlerRunner({"foo": "bar"}) assert runner.settings["foo"] == "bar" diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 88070513f..3f659ee0b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -18,7 +18,6 @@ import lxml.etree import pytest from testfixtures import LogCapture from w3lib.url import file_uri_to_path -from zope.interface import implementer import scrapy from scrapy import Spider, signals @@ -29,7 +28,6 @@ from scrapy.extensions.feedexport import ( FeedExporter, FeedSlot, FileFeedStorage, - IFeedStorage, apply_uri_params, ) from scrapy.utils.python import to_unicode @@ -83,7 +81,6 @@ class FailingBlockingFeedStorage(DummyBlockingFeedStorage): raise OSError("Cannot store") -@implementer(IFeedStorage) class LogOnStoreFileStorage: """ This storage logs inside `store` method. @@ -1300,7 +1297,6 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_storage_file_no_postprocessing(self): - @implementer(IFeedStorage) class Storage: def __init__(self, uri, *, feed_options=None): pass @@ -1322,7 +1318,6 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_storage_file_postprocessing(self): - @implementer(IFeedStorage) class Storage: def __init__(self, uri, *, feed_options=None): pass diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 6e0536110..3b50cd492 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -12,12 +12,11 @@ from urllib.parse import urljoin import lxml.etree import pytest from packaging.version import Version -from zope.interface.verify import verifyObject import scrapy from scrapy import Spider from scrapy.exceptions import NotConfigured -from scrapy.extensions.feedexport import FeedExporter, IFeedStorage, S3FeedStorage +from scrapy.extensions.feedexport import FeedExporter, S3FeedStorage from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler @@ -436,9 +435,6 @@ class TestBatchDeliveries(TestFeedExportBase): }, }, } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler(crawler, uri) - verifyObject(IFeedStorage, storage) class TestSpider(scrapy.Spider): name = "testspider" diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index b7009be62..82e2597a5 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -12,7 +12,6 @@ from urllib.parse import quote import pytest from testfixtures import LogCapture from w3lib.url import path_to_file_uri -from zope.interface.verify import verifyObject import scrapy from scrapy.extensions.feedexport import ( @@ -20,7 +19,6 @@ from scrapy.extensions.feedexport import ( FileFeedStorage, FTPFeedStorage, GCSFeedStorage, - IFeedStorage, S3FeedStorage, StdoutFeedStorage, ) @@ -55,11 +53,6 @@ class TestFileFeedStorage: finally: os.chdir(old_cwd) - def test_interface(self, tmp_path): - path = tmp_path / "file.txt" - st = FileFeedStorage(str(path)) - verifyObject(IFeedStorage, st) - @staticmethod def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None: storage = FileFeedStorage(str(path), feed_options=feed_options) @@ -115,7 +108,6 @@ class TestFTPFeedStorage: uri, feed_options=feed_options, ) - verifyObject(IFeedStorage, storage) spider = self.get_test_spider() file = storage.open(spider) file.write(content) @@ -259,7 +251,6 @@ class TestS3FeedStorage: bucket = "mybucket" key = "export.csv" storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}") - verifyObject(IFeedStorage, storage) file = mock.MagicMock() diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 81e49bb75..81840f7e0 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -5,14 +5,12 @@ from pathlib import Path from unittest import mock import pytest -from zope.interface.verify import verifyObject # ugly hack to avoid cyclic imports of scrapy.spiders when running this test # alone import scrapy from scrapy.crawler import CrawlerRunner from scrapy.http import Request -from scrapy.interfaces import ISpiderLoader from scrapy.settings import Settings from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader @@ -45,9 +43,6 @@ def spider_loader(spider_loader_env): class TestSpiderLoader: - def test_interface(self, spider_loader): - verifyObject(ISpiderLoader, spider_loader) - def test_list(self, spider_loader): assert set(spider_loader.list()) == { "spider1",