RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove non-Twisted uses of zope.interface (#7731)

This commit is contained in:
Andrey Rakhmatullin 2026-07-21 16:30:21 +05:00 committed by GitHub
parent e322905255
commit 394c2797f3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 87 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
docs/news.rst
View File

@ -3,6 +3,28 @@
Release notes
=============
Scrapy VERSION (unreleased)
---------------------------
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- The following runtime usage of zope.interface_ interfaces is removed:
- :class:`~scrapy.spiderloader.SpiderLoader` and
:class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked
as implementing the ``ISpiderLoader`` interface.
- :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the
configured spider loader implements the ``ISpiderLoader`` interface.
- :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`,
:class:`~scrapy.extensions.feedexport.FileFeedStorage` and
:class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer
marked as implementing the ``IFeedStorage`` interface.
(:issue:`6585`, :issue:`7731`)
.. _release-2.17.0:
Scrapy 2.17.0 (2026-07-07)
@ -2643,7 +2665,7 @@ Deprecation removals
(:issue:`6109`, :issue:`6116`)
- A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that
does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface
does not implement the ``ISpiderLoader`` interface
will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at
run time. Non-compliant classes have been triggering a deprecation warning
since Scrapy 1.0.0.

45
docs/topics/api.rst
View File

@ -172,46 +172,15 @@ SpiderLoader API
.. module:: scrapy.spiderloader
:synopsis: The spider loader
.. class:: SpiderLoader
Custom spider loaders can be employed by specifying their path in the
:setting:`SPIDER_LOADER_CLASS` project setting. They must implement
:class:`SpiderLoaderProtocol`.
This class is in charge of retrieving and handling the spider classes
defined across the project.
.. autoclass:: SpiderLoaderProtocol
:members:
Custom spider loaders can be employed by specifying their path in the
:setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement
the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an
errorless execution.
.. method:: from_settings(settings)
This class method is used by Scrapy to create an instance of the class.
It's called with the current project settings, and it loads the spiders
found recursively in the modules of the :setting:`SPIDER_MODULES`
setting.
:param settings: project settings
:type settings: :class:`~scrapy.settings.Settings` instance
.. method:: load(spider_name)
Get the Spider class with the given name. It'll look into the previously
loaded spiders for a spider class with name ``spider_name`` and will raise
a KeyError if not found.
:param spider_name: spider class name
:type spider_name: str
.. method:: list()
Get the names of the available spiders in the project.
.. method:: find_by_request(request)
List the spiders' names that can handle the given request. Will try to
match the request's url against the domains of the spiders.
:param request: queried request
:type request: :class:`~scrapy.Request` instance
.. autoclass:: SpiderLoader
:members:
.. autoclass:: DummySpiderLoader

2
scrapy/core/downloader/contextfactory.py
View File

@ -13,7 +13,6 @@ from twisted.internet.ssl import (
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface.declarations import implementer
from zope.interface.verify import verifyObject
from scrapy.core.downloader.tls import (
_TWISTED_VERSION_MAP,
@ -232,7 +231,6 @@ class _AcceptableProtocolsContextFactory:
# all of this with _ScrapyClientContextFactory.acceptableProtocols.
def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]):
verifyObject(IPolicyForHTTPS, context_factory)
self._wrapped_context_factory: Any = context_factory
self._acceptable_protocols: list[bytes] = acceptable_protocols

33
scrapy/extensions/feedexport.py
View File

@ -22,7 +22,7 @@ from urllib.parse import unquote, urlparse
from twisted.internet.defer import Deferred, DeferredList
from w3lib.url import file_uri_to_path
from zope.interface import Interface, implementer
from zope.interface import Interface
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
@ -115,25 +115,18 @@ class ItemFilter:
return True # accept all items by default
class IFeedStorage(Interface): # type: ignore[misc]
"""Interface that all Feed Storages must implement"""
class _IFeedStorage(Interface): # type: ignore[misc] # pragma: no cover
# pylint: disable=no-self-argument
def __init__(uri, *, feed_options=None): # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called
"""Initialize the storage with the parameters given in the URI and the
feed-specific options (see :setting:`FEEDS`)"""
def __init__(uri, *, feed_options=None): ... # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called
def open(spider): # type: ignore[no-untyped-def]
"""Open the storage for the given spider. It must return a file-like
object that will be used for the exporters"""
def open(spider): ... # type: ignore[no-untyped-def]
def store(file): # type: ignore[no-untyped-def]
"""Store the given file stream"""
def store(file): ... # type: ignore[no-untyped-def]
class FeedStorageProtocol(Protocol):
"""Reimplementation of ``IFeedStorage`` that can be used in type hints."""
"""Protocol that all Feed Storages must follow."""
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
"""Initialize the storage with the parameters given in the URI and the
@ -147,7 +140,6 @@ class FeedStorageProtocol(Protocol):
"""Store the given file stream"""
@implementer(IFeedStorage)
class BlockingFeedStorage(ABC):
def open(self, spider: Spider) -> IO[bytes]:
path = spider.crawler.settings["FEED_TEMPDIR"]
@ -164,7 +156,6 @@ class BlockingFeedStorage(ABC):
raise NotImplementedError
@implementer(IFeedStorage)
class StdoutFeedStorage:
def __init__(
self,
@ -191,7 +182,6 @@ class StdoutFeedStorage:
pass
@implementer(IFeedStorage)
class FileFeedStorage:
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri
@ -758,3 +748,14 @@ class FeedExporter:
feed_options.get("item_filter", ItemFilter)
)
return item_filter_class(feed_options)
def __getattr__(name: str) -> Any: # pragma: no cover
if name == "IFeedStorage":
warnings.warn(
"scrapy.extensions.feedexport.IFeedStorage is deprecated.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return _IFeedStorage
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

24
scrapy/interfaces.py
View File

@ -1,19 +1,23 @@
# pragma: no file cover
# pylint: disable=no-method-argument,no-self-argument
import warnings
from zope.interface import Interface
from scrapy.exceptions import ScrapyDeprecationWarning
warnings.warn(
"The scrapy.interfaces module is deprecated.",
ScrapyDeprecationWarning,
stacklevel=2,
)
class ISpiderLoader(Interface):
def from_settings(settings):
"""Return an instance of the class for the given settings"""
def from_settings(settings): ...
def load(spider_name):
"""Return the Spider class for the given spider name. If the spider
name is not found, it must raise a KeyError."""
def load(spider_name): ...
def list():
"""Return a list with the names of all spiders available in the
project"""
def list(): ...
def find_by_request(request):
"""Return the list of spiders names that can handle the given request"""
def find_by_request(request): ...

44
scrapy/spiderloader.py
View File

@ -5,10 +5,8 @@ import warnings
from collections import defaultdict
from typing import TYPE_CHECKING, Protocol, cast
from zope.interface import implementer
from zope.interface.verify import verifyClass
from scrapy.interfaces import ISpiderLoader
# working around https://github.com/sphinx-doc/sphinx/issues/10400
from scrapy import Request, Spider # noqa: TC001
from scrapy.utils.misc import load_object, walk_modules_iter
from scrapy.utils.spider import iter_spider_classes
@ -18,7 +16,6 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Request, Spider
from scrapy.settings import BaseSettings
@ -26,28 +23,31 @@ def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
"""Get SpiderLoader instance from settings"""
cls_path = settings.get("SPIDER_LOADER_CLASS")
loader_cls = load_object(cls_path)
verifyClass(ISpiderLoader, loader_cls)
return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()))
class SpiderLoaderProtocol(Protocol):
"""Protocol for spider loader implementations.
See :setting:`SPIDER_LOADER_CLASS`.
"""
@classmethod
def from_settings(cls, settings: BaseSettings) -> Self:
"""Return an instance of the class for the given settings"""
"""Return an instance of the class for the given settings."""
def load(self, spider_name: str) -> type[Spider]:
"""Return the Spider class for the given spider name. If the spider
name is not found, it must raise a KeyError."""
"""Return the spider class for the given spider name. If the spider
name is not found, it must raise a :exc:`KeyError`."""
def list(self) -> list[str]:
"""Return a list with the names of all spiders available in the
project"""
project."""
def find_by_request(self, request: Request) -> __builtins__.list[str]:
"""Return the list of spiders names that can handle the given request"""
"""Return the list of spiders names that can handle the given request."""
@implementer(ISpiderLoader)
class SpiderLoader:
"""
SpiderLoader is a class which locates and loads spiders
@ -106,12 +106,18 @@ class SpiderLoader:
@classmethod
def from_settings(cls, settings: BaseSettings) -> Self:
"""Create an instance of the class.
It's called with the current project settings, and it loads the spiders
found recursively in the modules of the :setting:`SPIDER_MODULES`
setting.
"""
return cls(settings)
def load(self, spider_name: str) -> type[Spider]:
"""
Return the Spider class for the given spider name. If the spider
name is not found, raise a KeyError.
"""Return the spider class for the given spider name.
If the spider name is not found, raise a :exc:`KeyError`.
"""
try:
return self._spiders[spider_name]
@ -121,19 +127,19 @@ class SpiderLoader:
def find_by_request(self, request: Request) -> list[str]:
"""
Return the list of spider names that can handle the given request.
It will try to match the request's url against the domains of
the spiders.
"""
return [
name for name, cls in self._spiders.items() if cls.handles_request(request)
]
def list(self) -> list[str]:
"""
Return a list with the names of all spiders available in the project.
"""
"""Return a list with the names of all spiders available in the project."""
return list(self._spiders.keys())
@implementer(ISpiderLoader)
class DummySpiderLoader:
"""A dummy spider loader that does not load any spiders."""

20
tests/test_crawler_runners.py
View File

@ -4,7 +4,6 @@ import logging
from typing import TYPE_CHECKING
import pytest
from zope.interface.exceptions import MultipleInvalid
from scrapy.crawler import (
AsyncCrawlerProcess,
@ -14,7 +13,6 @@ from scrapy.crawler import (
CrawlerRunner,
CrawlerRunnerBase,
)
from scrapy.settings import Settings
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_reactor_settings
@ -33,15 +31,6 @@ class SpiderLoaderWithWrongInterface:
class TestCrawlerRunner:
def test_spider_manager_verify_interface(self) -> None:
settings = Settings(
{
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
}
)
with pytest.raises(MultipleInvalid):
CrawlerRunner(settings)
def test_crawler_runner_accepts_dict(self) -> None:
runner = CrawlerRunner({"foo": "bar"})
assert runner.settings["foo"] == "bar"
@ -53,15 +42,6 @@ class TestCrawlerRunner:
class TestAsyncCrawlerRunner:
def test_spider_manager_verify_interface(self) -> None:
settings = Settings(
{
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
}
)
with pytest.raises(MultipleInvalid):
AsyncCrawlerRunner(settings)
def test_crawler_runner_accepts_dict(self) -> None:
runner = AsyncCrawlerRunner({"foo": "bar"})
assert runner.settings["foo"] == "bar"

5
tests/test_feedexport.py
View File

@ -18,7 +18,6 @@ import lxml.etree
import pytest
from testfixtures import LogCapture
from w3lib.url import file_uri_to_path
from zope.interface import implementer
import scrapy
from scrapy import Spider, signals
@ -29,7 +28,6 @@ from scrapy.extensions.feedexport import (
FeedExporter,
FeedSlot,
FileFeedStorage,
IFeedStorage,
apply_uri_params,
)
from scrapy.utils.python import to_unicode
@ -83,7 +81,6 @@ class FailingBlockingFeedStorage(DummyBlockingFeedStorage):
raise OSError("Cannot store")
@implementer(IFeedStorage)
class LogOnStoreFileStorage:
"""
This storage logs inside `store` method.
@ -1300,7 +1297,6 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_storage_file_no_postprocessing(self):
@implementer(IFeedStorage)
class Storage:
def __init__(self, uri, *, feed_options=None):
pass
@ -1322,7 +1318,6 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_storage_file_postprocessing(self):
@implementer(IFeedStorage)
class Storage:
def __init__(self, uri, *, feed_options=None):
pass

6
tests/test_feedexport_batch.py
View File

@ -12,12 +12,11 @@ from urllib.parse import urljoin
import lxml.etree
import pytest
from packaging.version import Version
from zope.interface.verify import verifyObject
import scrapy
from scrapy import Spider
from scrapy.exceptions import NotConfigured
from scrapy.extensions.feedexport import FeedExporter, IFeedStorage, S3FeedStorage
from scrapy.extensions.feedexport import FeedExporter, S3FeedStorage
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
@ -436,9 +435,6 @@ class TestBatchDeliveries(TestFeedExportBase):
},
},
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(crawler, uri)
verifyObject(IFeedStorage, storage)
class TestSpider(scrapy.Spider):
name = "testspider"

9
tests/test_feedexport_storages.py
View File

@ -12,7 +12,6 @@ from urllib.parse import quote
import pytest
from testfixtures import LogCapture
from w3lib.url import path_to_file_uri
from zope.interface.verify import verifyObject
import scrapy
from scrapy.extensions.feedexport import (
@ -20,7 +19,6 @@ from scrapy.extensions.feedexport import (
FileFeedStorage,
FTPFeedStorage,
GCSFeedStorage,
IFeedStorage,
S3FeedStorage,
StdoutFeedStorage,
)
@ -55,11 +53,6 @@ class TestFileFeedStorage:
finally:
os.chdir(old_cwd)
def test_interface(self, tmp_path):
path = tmp_path / "file.txt"
st = FileFeedStorage(str(path))
verifyObject(IFeedStorage, st)
@staticmethod
def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None:
storage = FileFeedStorage(str(path), feed_options=feed_options)
@ -115,7 +108,6 @@ class TestFTPFeedStorage:
uri,
feed_options=feed_options,
)
verifyObject(IFeedStorage, storage)
spider = self.get_test_spider()
file = storage.open(spider)
file.write(content)
@ -259,7 +251,6 @@ class TestS3FeedStorage:
bucket = "mybucket"
key = "export.csv"
storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}")
verifyObject(IFeedStorage, storage)
file = mock.MagicMock()

5
tests/test_spiderloader/__init__.py
View File

@ -5,14 +5,12 @@ from pathlib import Path
from unittest import mock
import pytest
from zope.interface.verify import verifyObject
# ugly hack to avoid cyclic imports of scrapy.spiders when running this test
# alone
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.http import Request
from scrapy.interfaces import ISpiderLoader
from scrapy.settings import Settings
from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader
@ -45,9 +43,6 @@ def spider_loader(spider_loader_env):
class TestSpiderLoader:
def test_interface(self, spider_loader):
verifyObject(ISpiderLoader, spider_loader)
def test_list(self, spider_loader):
assert set(spider_loader.list()) == {
"spider1",