diff --git a/docs/news.rst b/docs/news.rst index 63f0d69e4..3964722c7 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,28 @@ Release notes ============= +Scrapy VERSION (unreleased) +--------------------------- + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The following runtime usage of zope.interface_ interfaces is removed: + + - :class:`~scrapy.spiderloader.SpiderLoader` and + :class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked + as implementing the ``ISpiderLoader`` interface. + + - :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the + configured spider loader implements the ``ISpiderLoader`` interface. + + - :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`, + - :class:`~scrapy.extensions.feedexport.FileFeedStorage` and + - :class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer + marked as implementing the ``IFeedStorage`` interface. + + (:issue:`6585`, :issue:`TBD`) + .. _release-2.17.0: Scrapy 2.17.0 (2026-07-07) @@ -2643,7 +2665,7 @@ Deprecation removals (:issue:`6109`, :issue:`6116`) - A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that - does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface + does not implement the ``ISpiderLoader`` interface will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at run time. Non-compliant classes have been triggering a deprecation warning since Scrapy 1.0.0. diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 19082d9d7..c80bbd539 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -178,9 +178,7 @@ SpiderLoader API defined across the project. Custom spider loaders can be employed by specifying their path in the - :setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement - the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an - errorless execution. + :setting:`SPIDER_LOADER_CLASS` project setting. .. method:: from_settings(settings) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index a934cbbc7..ef948997d 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -13,7 +13,6 @@ from twisted.internet.ssl import ( from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer -from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import ( _TWISTED_VERSION_MAP, @@ -232,7 +231,6 @@ class _AcceptableProtocolsContextFactory: # all of this with _ScrapyClientContextFactory.acceptableProtocols. def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]): - verifyObject(IPolicyForHTTPS, context_factory) self._wrapped_context_factory: Any = context_factory self._acceptable_protocols: list[bytes] = acceptable_protocols diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fa708a2e1..9c95e3331 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -22,7 +22,7 @@ from urllib.parse import unquote, urlparse from twisted.internet.defer import Deferred, DeferredList from w3lib.url import file_uri_to_path -from zope.interface import Interface, implementer +from zope.interface import Interface from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning @@ -88,25 +88,18 @@ class ItemFilter: return True # accept all items by default -class IFeedStorage(Interface): # type: ignore[misc] - """Interface that all Feed Storages must implement""" - +class _IFeedStorage(Interface): # type: ignore[misc] # pragma: no cover # pylint: disable=no-self-argument - def __init__(uri, *, feed_options=None): # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called - """Initialize the storage with the parameters given in the URI and the - feed-specific options (see :setting:`FEEDS`)""" + def __init__(uri, *, feed_options=None): ... # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called - def open(spider): # type: ignore[no-untyped-def] - """Open the storage for the given spider. It must return a file-like - object that will be used for the exporters""" + def open(spider): ... # type: ignore[no-untyped-def] - def store(file): # type: ignore[no-untyped-def] - """Store the given file stream""" + def store(file): ... # type: ignore[no-untyped-def] class FeedStorageProtocol(Protocol): - """Reimplementation of ``IFeedStorage`` that can be used in type hints.""" + """Protocol that all Feed Storages must follow.""" def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): """Initialize the storage with the parameters given in the URI and the @@ -120,7 +113,6 @@ class FeedStorageProtocol(Protocol): """Store the given file stream""" -@implementer(IFeedStorage) class BlockingFeedStorage(ABC): def open(self, spider: Spider) -> IO[bytes]: path = spider.crawler.settings["FEED_TEMPDIR"] @@ -137,7 +129,6 @@ class BlockingFeedStorage(ABC): raise NotImplementedError -@implementer(IFeedStorage) class StdoutFeedStorage: def __init__( self, @@ -164,7 +155,6 @@ class StdoutFeedStorage: pass -@implementer(IFeedStorage) class FileFeedStorage: def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri @@ -731,3 +721,14 @@ class FeedExporter: feed_options.get("item_filter", ItemFilter) ) return item_filter_class(feed_options) + + +def __getattr__(name: str) -> Any: # pragma: no cover + if name == "IFeedStorage": + warnings.warn( + "scrapy.extensions.feedexport.IFeedStorage is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _IFeedStorage + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index b4f1d9394..34d7bdd82 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,19 +1,23 @@ +# pragma: no file cover # pylint: disable=no-method-argument,no-self-argument +import warnings from zope.interface import Interface +from scrapy.exceptions import ScrapyDeprecationWarning + +warnings.warn( + "The scrapy.interfaces module is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, +) + class ISpiderLoader(Interface): - def from_settings(settings): - """Return an instance of the class for the given settings""" + def from_settings(settings): ... - def load(spider_name): - """Return the Spider class for the given spider name. If the spider - name is not found, it must raise a KeyError.""" + def load(spider_name): ... - def list(): - """Return a list with the names of all spiders available in the - project""" + def list(): ... - def find_by_request(request): - """Return the list of spiders names that can handle the given request""" + def find_by_request(request): ... diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 8c980fd46..d98e1a5a0 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -5,10 +5,6 @@ import warnings from collections import defaultdict from typing import TYPE_CHECKING, Protocol, cast -from zope.interface import implementer -from zope.interface.verify import verifyClass - -from scrapy.interfaces import ISpiderLoader from scrapy.utils.misc import load_object, walk_modules_iter from scrapy.utils.spider import iter_spider_classes @@ -26,7 +22,6 @@ def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) - verifyClass(ISpiderLoader, loader_cls) return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())) @@ -47,7 +42,6 @@ class SpiderLoaderProtocol(Protocol): """Return the list of spiders names that can handle the given request""" -@implementer(ISpiderLoader) class SpiderLoader: """ SpiderLoader is a class which locates and loads spiders @@ -133,7 +127,6 @@ class SpiderLoader: return list(self._spiders.keys()) -@implementer(ISpiderLoader) class DummySpiderLoader: """A dummy spider loader that does not load any spiders.""" diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6a1eea027..5106c890e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -10,7 +10,6 @@ from typing import TYPE_CHECKING, Any, ClassVar from unittest.mock import MagicMock import pytest -from zope.interface.exceptions import MultipleInvalid import scrapy from scrapy import Spider @@ -586,21 +585,7 @@ class TestCrawlerLogging: assert "debug message" in logged -class SpiderLoaderWithWrongInterface: - def unneeded_method(self) -> None: - pass - - class TestCrawlerRunner(TestBaseCrawler): - def test_spider_manager_verify_interface(self) -> None: - settings = Settings( - { - "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, - } - ) - with pytest.raises(MultipleInvalid): - CrawlerRunner(settings) - def test_crawler_runner_accepts_dict(self) -> None: runner = CrawlerRunner({"foo": "bar"}) assert runner.settings["foo"] == "bar" @@ -612,15 +597,6 @@ class TestCrawlerRunner(TestBaseCrawler): class TestAsyncCrawlerRunner(TestBaseCrawler): - def test_spider_manager_verify_interface(self) -> None: - settings = Settings( - { - "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, - } - ) - with pytest.raises(MultipleInvalid): - AsyncCrawlerRunner(settings) - def test_crawler_runner_accepts_dict(self) -> None: runner = AsyncCrawlerRunner({"foo": "bar"}) assert runner.settings["foo"] == "bar" diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 7d751f188..557c4bb1b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -20,7 +20,6 @@ import lxml.etree import pytest from testfixtures import LogCapture from w3lib.url import file_uri_to_path -from zope.interface import implementer import scrapy from scrapy import Spider, signals @@ -31,7 +30,6 @@ from scrapy.extensions.feedexport import ( FeedExporter, FeedSlot, FileFeedStorage, - IFeedStorage, ) from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler @@ -91,7 +89,6 @@ class FailingBlockingFeedStorage(DummyBlockingFeedStorage): raise OSError("Cannot store") -@implementer(IFeedStorage) class LogOnStoreFileStorage: """ This storage logs inside `store` method. @@ -1234,7 +1231,6 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_storage_file_no_postprocessing(self): - @implementer(IFeedStorage) class Storage: def __init__(self, uri, *, feed_options=None): pass @@ -1256,7 +1252,6 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_storage_file_postprocessing(self): - @implementer(IFeedStorage) class Storage: def __init__(self, uri, *, feed_options=None): pass diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 469985288..a7547a30c 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -12,12 +12,11 @@ from urllib.parse import urljoin import lxml.etree import pytest from packaging.version import Version -from zope.interface.verify import verifyObject import scrapy from scrapy import Spider from scrapy.exceptions import NotConfigured -from scrapy.extensions.feedexport import FeedExporter, IFeedStorage, S3FeedStorage +from scrapy.extensions.feedexport import FeedExporter, S3FeedStorage from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler @@ -435,9 +434,6 @@ class TestBatchDeliveries(TestFeedExportBase): }, }, } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler(crawler, uri) - verifyObject(IFeedStorage, storage) class TestSpider(scrapy.Spider): name = "testspider" diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index 11e79775f..cddbc5f97 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -12,7 +12,6 @@ from urllib.parse import quote import pytest from testfixtures import LogCapture from w3lib.url import path_to_file_uri -from zope.interface.verify import verifyObject import scrapy from scrapy.extensions.feedexport import ( @@ -20,7 +19,6 @@ from scrapy.extensions.feedexport import ( FileFeedStorage, FTPFeedStorage, GCSFeedStorage, - IFeedStorage, S3FeedStorage, StdoutFeedStorage, ) @@ -71,11 +69,6 @@ class TestFileFeedStorage: finally: os.chdir(old_cwd) - def test_interface(self, tmp_path): - path = tmp_path / "file.txt" - st = FileFeedStorage(str(path)) - verifyObject(IFeedStorage, st) - @staticmethod def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None: storage = FileFeedStorage(str(path), feed_options=feed_options) @@ -131,7 +124,6 @@ class TestFTPFeedStorage: uri, feed_options=feed_options, ) - verifyObject(IFeedStorage, storage) spider = self.get_test_spider() file = storage.open(spider) file.write(content) @@ -275,7 +267,6 @@ class TestS3FeedStorage: bucket = "mybucket" key = "export.csv" storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}") - verifyObject(IFeedStorage, storage) file = mock.MagicMock() diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d85942f90..8974cec89 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -5,14 +5,12 @@ from pathlib import Path from unittest import mock import pytest -from zope.interface.verify import verifyObject # ugly hack to avoid cyclic imports of scrapy.spiders when running this test # alone import scrapy from scrapy.crawler import CrawlerRunner from scrapy.http import Request -from scrapy.interfaces import ISpiderLoader from scrapy.settings import Settings from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader @@ -45,9 +43,6 @@ def spider_loader(spider_loader_env): class TestSpiderLoader: - def test_interface(self, spider_loader): - verifyObject(ISpiderLoader, spider_loader) - def test_list(self, spider_loader): assert set(spider_loader.list()) == { "spider1",