RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Replace PyDispatcher with blinker

This commit is contained in:
Adrian Chaves 2026-08-08 15:31:42 +02:00
parent 1bd839b57d
commit 7289af962b
10 changed files with 545 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/intro/install.rst
View File

@ -278,10 +278,6 @@ issues besides installing build dependencies.
Installing Scrapy with PyPy on Windows is not tested.
You can check that Scrapy is installed correctly by running ``scrapy bench``.
If this command gives errors such as
``TypeError: ... got 2 unexpected keyword arguments``, this means
that the ``PyPyDispatcher`` dependency wasn't installed. To fix this issue, run
``pip install 'PyPyDispatcher>=2.1.0'``.
.. _intro-install-troubleshooting:

5
pyproject.toml
View File

@ -8,6 +8,7 @@ dynamic = ["version"]
description = "A high-level Web Crawling and Web Scraping framework"
dependencies = [
"Twisted>=21.7.0",
"blinker>=1.7.0",
"cryptography>=37.0.0",
"cssselect>=0.9.1",
"defusedxml>=0.7.1",
@ -23,9 +24,6 @@ dependencies = [
"tldextract",
"w3lib>=1.17.0",
"zope.interface>=5.1.0",
# Platform-specific dependencies
'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',
'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"',
]
classifiers = [
"Development Status :: 5 - Production/Stable",
@ -187,7 +185,6 @@ module = [
"brotli",
"brotlicffi",
"google.*",
"pydispatch.*",
"pyftpdlib.*",
"pytest_twisted",
"robotexclusionrulesparser",

8
scrapy/signalmanager.py
View File

@ -3,16 +3,16 @@ from __future__ import annotations
import warnings
from typing import Any
from pydispatch import dispatcher
from twisted.internet.defer import Deferred
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils import _signal_registry
from scrapy.utils import signal as _signal
from scrapy.utils.defer import maybe_deferred_to_future
class SignalManager:
def __init__(self, sender: Any = dispatcher.Anonymous):
def __init__(self, sender: Any = _signal_registry.Anonymous):
self.sender: Any = sender
def connect(self, receiver: Any, signal: Any, **kwargs: Any) -> None:
@ -30,7 +30,7 @@ class SignalManager:
:type signal: object
"""
kwargs.setdefault("sender", self.sender)
dispatcher.connect(receiver, signal, **kwargs)
_signal_registry.connect(receiver, signal, **kwargs)
def disconnect(self, receiver: Any, signal: Any, **kwargs: Any) -> None:
"""
@ -39,7 +39,7 @@ class SignalManager:
are the same.
"""
kwargs.setdefault("sender", self.sender)
dispatcher.disconnect(receiver, signal, **kwargs)
_signal_registry.disconnect(receiver, signal, **kwargs)
def send_catch_log(self, signal: Any, **kwargs: Any) -> list[tuple[Any, Any]]:
"""

28
scrapy/signals.py
View File

@ -27,3 +27,31 @@ item_dropped = object()
item_error = object()
feed_slot_closed = object()
feed_exporter_closed = object()
#: Arguments that each signal sends, used to catch handlers that declare an
#: argument their signal never sends. ``signal`` and ``sender`` are omitted
#: because every signal sends them.
_signal_args: dict[object, frozenset[str]] = {
engine_started: frozenset(),
engine_stopped: frozenset(),
scheduler_empty: frozenset(),
spider_opened: frozenset({"spider"}),
spider_idle: frozenset({"spider"}),
spider_closed: frozenset({"spider", "reason"}),
spider_error: frozenset({"failure", "response", "spider"}),
memusage_warning_reached: frozenset(),
request_scheduled: frozenset({"request", "spider"}),
request_dropped: frozenset({"request", "spider"}),
request_reached_downloader: frozenset({"request", "spider"}),
request_left_downloader: frozenset({"request", "spider"}),
response_received: frozenset({"response", "request", "spider"}),
response_downloaded: frozenset({"response", "request", "spider"}),
headers_received: frozenset({"headers", "body_length", "request", "spider"}),
bytes_received: frozenset({"data", "request", "spider"}),
robots_parsed: frozenset({"robotparser", "request"}),
item_scraped: frozenset({"item", "response", "spider"}),
item_dropped: frozenset({"item", "response", "spider", "exception"}),
item_error: frozenset({"item", "response", "spider", "failure"}),
feed_slot_closed: frozenset({"slot"}),
feed_exporter_closed: frozenset(),
}

203
scrapy/utils/_signal_registry.py Normal file
View File

@ -0,0 +1,203 @@
from __future__ import annotations
import inspect
import warnings
from typing import TYPE_CHECKING
from typing import Any as TypingAny
from weakref import WeakKeyDictionary
from blinker import ANY, Signal
from scrapy import signals as _builtin_signals
from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
from collections.abc import Callable
#: Sender that matches receivers connected to any sender.
Any = ANY
#: Default sender of signals sent without one.
Anonymous = object()
#: Arguments that every signal sends, on top of its own.
_COMMON_ARGS = frozenset({"signal", "sender"})
class _OrderedSet(dict[TypingAny, TypingAny]):
"""Insertion-ordered set covering the subset of the ``set`` API that
:attr:`blinker.Signal.set_class` requires.
Receivers run in the order they were connected, which components rely on:
:class:`~scrapy.extensions.logcount.LogCount` and
:class:`~scrapy.extensions.logstats.LogStats` share a priority, so only
connection order puts the log counter in place before the first log message.
"""
def add(self, item: TypingAny) -> None:
self[item] = None
def discard(self, item: TypingAny) -> None:
self.pop(item, None)
def copy(self) -> _OrderedSet:
return _OrderedSet(self)
def __or__(self, other: TypingAny) -> _OrderedSet:
return _OrderedSet({**self, **other})
class _Signal(Signal):
set_class = _OrderedSet # type: ignore[assignment]
# Signals are usually module-level constants, but a signal that goes out of scope
# should not keep its receivers alive. Signals that cannot be weakly referenced,
# such as strings, are keyed by value instead.
_signals: WeakKeyDictionary[TypingAny, _Signal] = WeakKeyDictionary()
_signals_by_value: dict[TypingAny, _Signal] = {}
def _signal_for(signal: TypingAny) -> _Signal:
try:
return _signals[signal]
except KeyError:
obj = _signals[signal] = _Signal()
return obj
except TypeError:
pass
try:
return _signals_by_value[signal]
except KeyError:
obj = _signals_by_value[signal] = _Signal()
return obj
# Keyed by the underlying function rather than by the receiver, so that a bound
# method does not keep its instance alive, and so that every instance of a
# component shares one entry.
_accepted_args: dict[TypingAny, frozenset[str] | None] = {}
def _cache_key(receiver: TypingAny) -> TypingAny:
func = getattr(receiver, "__func__", None)
if func is not None:
return func
if inspect.isroutine(receiver):
return receiver
return type(receiver)
def _accepted(receiver: TypingAny) -> frozenset[str] | None:
"""Return the names of the arguments *receiver* accepts, or ``None`` if it
accepts arbitrary keyword arguments.
"""
key = _cache_key(receiver)
try:
return _accepted_args[key]
except KeyError:
pass
except TypeError:
return _introspect(receiver)
names = _introspect(receiver)
_accepted_args[key] = names
return names
def _introspect(receiver: TypingAny) -> frozenset[str] | None:
try:
parameters = inspect.signature(receiver).parameters.values()
except (TypeError, ValueError):
# Callables that cannot be introspected, e.g. some built-ins.
return None
if any(p.kind is p.VAR_KEYWORD for p in parameters):
return None
return frozenset(
p.name
for p in parameters
if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY)
)
def _warn_unknown_args(receiver: TypingAny, signal: TypingAny) -> None:
sent = _builtin_signals._signal_args.get(signal)
if sent is None:
return
accepted = _accepted(receiver)
if accepted is None:
return
unknown = accepted - sent - _COMMON_ARGS
if not unknown:
return
names = ", ".join(sorted(unknown))
warnings.warn(
f"Signal handler {global_object_name(receiver)} declares"
f" {names}, which {_signal_name(signal)} does not send. Handlers"
f" only receive the arguments of their signal, so those arguments always"
f" get their default value.",
stacklevel=4,
)
def _signal_name(signal: TypingAny) -> str:
for name, value in vars(_builtin_signals).items():
if value is signal:
return f"scrapy.signals.{name}"
return repr(signal)
def connect(
receiver: Callable[..., TypingAny],
signal: TypingAny,
sender: TypingAny = Any,
weak: bool = True,
) -> None:
if signal is Any:
raise ValueError(
"Connecting a receiver to every signal at once is not supported."
" Connect it to each signal that it handles instead."
)
_warn_unknown_args(receiver, signal)
_signal_for(signal).connect(receiver, sender=sender, weak=weak)
def disconnect(
receiver: Callable[..., TypingAny],
signal: TypingAny,
sender: TypingAny = Any,
) -> None:
_signal_for(signal).disconnect(receiver, sender=sender)
def receivers(signal: TypingAny, sender: TypingAny) -> list[Callable[..., TypingAny]]:
"""Return the live receivers of *signal* for *sender*, in connection order."""
return list(_signal_for(signal).receivers_for(sender))
def apply(
receiver: Callable[..., TypingAny],
*arguments: TypingAny,
**named: TypingAny,
) -> TypingAny:
"""Call *receiver* with *arguments* and the entries of *named* that it accepts."""
accepted = _accepted(receiver)
if accepted is not None:
if arguments:
accepted = accepted - _positional_names(receiver, len(arguments))
named = {k: v for k, v in named.items() if k in accepted}
return receiver(*arguments, **named)
def _positional_names(receiver: TypingAny, count: int) -> frozenset[str]:
"""Return the names of the first *count* parameters of *receiver*, which
positional arguments bind and keyword arguments therefore must not repeat.
"""
try:
parameters = list(inspect.signature(receiver).parameters.values())
except (TypeError, ValueError):
return frozenset()
return frozenset(
p.name
for p in parameters[:count]
if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)
)

33
scrapy/utils/signal.py
View File

@ -9,18 +9,11 @@ from collections.abc import Awaitable, Callable, Generator, Sequence
from typing import Any as TypingAny
from typing import cast
from pydispatch.dispatcher import (
Anonymous,
Any,
disconnect,
getAllReceivers,
liveReceivers,
)
from pydispatch.robustapply import robustApply
from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
from twisted.python.failure import Failure
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
from scrapy.utils._signal_registry import Anonymous, Any, apply, disconnect, receivers
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.defer import (
_maybeDeferred_coro,
@ -39,18 +32,18 @@ def send_catch_log(
*arguments: TypingAny,
**named: TypingAny,
) -> list[tuple[TypingAny, TypingAny]]:
"""Like ``pydispatch.robust.sendRobust()`` but it also logs errors and returns
Failures instead of exceptions.
"""Send *signal*, logging any error raised by a handler and returning Failures
instead of raising.
"""
dont_log = named.pop("dont_log", ())
dont_log = tuple(dont_log) if isinstance(dont_log, Sequence) else (dont_log,)
dont_log += (StopDownload,)
spider = named.get("spider")
responses: list[tuple[TypingAny, TypingAny]] = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
for receiver in receivers(signal, sender):
result: TypingAny
try:
response = robustApply(
response = apply(
receiver, *arguments, signal=signal, sender=sender, **named
)
if isinstance(response, Deferred):
@ -114,9 +107,9 @@ def _send_catch_log_deferred(
dont_log = named.pop("dont_log", None)
spider = named.get("spider")
dfds: list[Deferred[tuple[TypingAny, TypingAny]]] = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
for receiver in receivers(signal, sender):
d: Deferred[TypingAny] = _maybeDeferred_coro(
robustApply,
apply,
True,
receiver,
*arguments,
@ -181,17 +174,15 @@ async def _send_catch_log_asyncio(
dont_log = tuple(dont_log) if isinstance(dont_log, Sequence) else (dont_log,)
spider = named.get("spider")
handlers: list[Awaitable[TypingAny]] = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
for receiver in receivers(signal, sender):
async def handler(
receiver: Callable[..., Any],
) -> tuple[Callable[..., Any], TypingAny]:
receiver: Callable[..., TypingAny],
) -> tuple[Callable[..., TypingAny], TypingAny]:
result: TypingAny
try:
result = await ensure_awaitable(
robustApply(
receiver, *arguments, signal=signal, sender=sender, **named
),
apply(receiver, *arguments, signal=signal, sender=sender, **named),
_warn=global_object_name(receiver),
)
except dont_log as ex: # pylint: disable=catching-non-exception
@ -218,5 +209,5 @@ def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None:
"""Disconnect all signal handlers. Useful for cleaning up after running
tests.
"""
for receiver in liveReceivers(getAllReceivers(sender, signal)):
for receiver in receivers(signal, sender):
disconnect(receiver, signal=signal, sender=sender)

2
tests/test_utils_signal.py
View File

@ -4,10 +4,10 @@ import asyncio
from typing import TYPE_CHECKING, Any
import pytest
from pydispatch import dispatcher
from twisted.internet import defer
from twisted.python.failure import Failure
from scrapy.utils import _signal_registry as dispatcher
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
from scrapy.utils.signal import (

293
tests/test_utils_signal_registry.py Normal file
View File

@ -0,0 +1,293 @@
from __future__ import annotations
import gc
import weakref
from collections import Counter
from functools import partial
from typing import TYPE_CHECKING, Any
import pytest
from scrapy import signals
from scrapy.signalmanager import SignalManager
from scrapy.utils import _signal_registry as registry
if TYPE_CHECKING:
from collections.abc import Callable
class TestArgumentDelivery:
def test_handler_gets_only_the_arguments_it_declares(self) -> None:
received: dict[str, Any] = {}
def handler(spider: Any = None) -> None:
received.update(spider=spider)
signal = object()
sm = SignalManager(object())
sm.connect(handler, signal)
sm.send_catch_log(signal, spider="SPIDER")
assert received == {"spider": "SPIDER"}
def test_handler_with_kwargs_gets_every_argument(self) -> None:
received: dict[str, Any] = {}
def handler(**kwargs: Any) -> None:
received.update(kwargs)
signal = object()
sm = SignalManager(object())
sm.connect(handler, signal)
sm.send_catch_log(signal, spider="SPIDER")
assert received["spider"] == "SPIDER"
assert received["signal"] is signal
def test_keyword_only_handler(self) -> None:
received: dict[str, Any] = {}
def handler(*, spider: Any = None) -> None:
received.update(spider=spider)
signal = object()
sm = SignalManager(object())
sm.connect(handler, signal)
sm.send_catch_log(signal, spider="SPIDER")
assert received == {"spider": "SPIDER"}
def test_partial_handler(self) -> None:
received: dict[str, Any] = {}
def handler(prefix: str, spider: Any = None) -> None:
received.update(spider=f"{prefix}{spider}")
bound = partial(handler, "p-")
signal = object()
sm = SignalManager(object())
sm.connect(bound, signal)
sm.send_catch_log(signal, spider="SPIDER")
assert received == {"spider": "p-SPIDER"}
def test_callable_object_handler(self) -> None:
class Handler:
def __init__(self) -> None:
self.spider: Any = None
def __call__(self, spider: Any = None) -> None:
self.spider = spider
handler = Handler()
signal = object()
sm = SignalManager(object())
sm.connect(handler, signal)
sm.send_catch_log(signal, spider="SPIDER")
assert handler.spider == "SPIDER"
def _appending_handlers(calls: list[int], count: int) -> list[Callable[..., None]]:
def make(n: int) -> Callable[..., None]:
def handler(**kwargs: Any) -> None:
calls.append(n)
return handler
return [make(n) for n in range(count)]
class TestDispatchOrder:
def test_handlers_run_in_connection_order(self) -> None:
calls: list[int] = []
signal = object()
sm = SignalManager(object())
handlers = _appending_handlers(calls, 10)
for handler in handlers:
sm.connect(handler, signal)
sm.send_catch_log(signal)
assert calls == list(range(10))
def test_order_survives_disconnect(self) -> None:
calls: list[int] = []
signal = object()
sm = SignalManager(object())
handlers = _appending_handlers(calls, 5)
for handler in handlers:
sm.connect(handler, signal)
sm.disconnect(handlers[2], signal)
sm.send_catch_log(signal)
assert calls == [0, 1, 3, 4]
class TestDisconnect:
def test_disconnected_handler_stops_running(self) -> None:
calls: list[str] = []
signal = object()
sm = SignalManager(object())
def kept(**kwargs: Any) -> None:
calls.append("kept")
def dropped(**kwargs: Any) -> None:
calls.append("dropped")
sm.connect(kept, signal)
sm.connect(dropped, signal)
sm.disconnect(dropped, signal)
sm.send_catch_log(signal)
assert calls == ["kept"]
def test_every_handler_runs(self) -> None:
calls: set[str] = set()
signal = object()
sm = SignalManager(object())
handlers = []
for name in ("a", "b", "c"):
def handler(_name: str = name, **kwargs: Any) -> None:
calls.add(_name)
handlers.append(handler)
sm.connect(handler, signal)
sm.send_catch_log(signal)
assert calls == {"a", "b", "c"}
class TestSenderIsolation:
def test_handlers_only_run_for_their_sender(self) -> None:
calls: list[str] = []
def first_handler(**kwargs: Any) -> None:
calls.append("first")
def second_handler(**kwargs: Any) -> None:
calls.append("second")
signal = object()
first, second = SignalManager(object()), SignalManager(object())
first.connect(first_handler, signal)
second.connect(second_handler, signal)
first.send_catch_log(signal)
assert calls == ["first"]
class TestUnknownArgumentWarning:
def test_warns_about_an_argument_the_signal_does_not_send(self) -> None:
def handler(reason: Any = None) -> None:
pass
sm = SignalManager(object())
with pytest.warns(UserWarning, match="declares reason"):
sm.connect(handler, signals.spider_opened)
def test_warning_names_the_signal(self) -> None:
def handler(item: Any = None) -> None:
pass
sm = SignalManager(object())
with pytest.warns(UserWarning, match=r"scrapy\.signals\.spider_opened"):
sm.connect(handler, signals.spider_opened)
def test_lists_every_unknown_argument(self) -> None:
def handler(cheese: Any = None, ham: Any = None) -> None:
pass
sm = SignalManager(object())
with pytest.warns(UserWarning, match="declares cheese, ham"):
sm.connect(handler, signals.spider_opened)
@pytest.mark.filterwarnings("error")
def test_no_warning_for_a_matching_handler(self) -> None:
def handler(spider: Any = None, signal: Any = None, sender: Any = None) -> None:
pass
SignalManager(object()).connect(handler, signals.spider_opened)
@pytest.mark.filterwarnings("error")
def test_no_warning_for_a_kwargs_handler(self) -> None:
def handler(anything: Any = None, **kwargs: Any) -> None:
pass
SignalManager(object()).connect(handler, signals.spider_opened)
@pytest.mark.filterwarnings("error")
def test_no_warning_for_an_unknown_signal(self) -> None:
def handler(whatever: Any = None) -> None:
pass
SignalManager(object()).connect(handler, object())
class TestSignalArgs:
def test_every_signal_declares_its_arguments(self) -> None:
defined = {
value
for name, value in vars(signals).items()
if not name.startswith("_") and isinstance(value, object)
}
documented = set(signals._signal_args)
assert defined - documented == set()
class TestReceiverCache:
def test_bound_method_handler_is_not_kept_alive(self) -> None:
class Component:
def handler(self, spider: Any = None) -> None:
pass
component = Component()
ref = weakref.ref(component)
signal = object()
sm = SignalManager(object())
sm.connect(component.handler, signal)
del component
gc.collect()
assert ref() is None
assert registry.receivers(signal, sm.sender) == []
class TestMutationDuringDispatch:
def test_handler_can_disconnect_itself(self) -> None:
calls: list[str] = []
signal = object()
sm = SignalManager(object())
def handler(**kwargs: Any) -> None:
calls.append("handler")
sm.disconnect(handler, signal)
def other(**kwargs: Any) -> None:
calls.append("other")
sm.connect(handler, signal)
sm.connect(other, signal)
sm.send_catch_log(signal)
sm.send_catch_log(signal)
assert Counter(calls) == {"handler": 1, "other": 2}
def test_handler_can_connect_another_handler(self) -> None:
calls: list[str] = []
signal = object()
sm = SignalManager(object())
added: list[Callable[..., None]] = []
def late(**kwargs: Any) -> None:
calls.append("late")
def handler(**kwargs: Any) -> None:
calls.append("handler")
if not added:
added.append(late)
sm.connect(late, signal)
sm.connect(handler, signal)
sm.send_catch_log(signal)
assert Counter(calls) == {"handler": 1}
sm.send_catch_log(signal)
assert Counter(calls) == {"handler": 2, "late": 1}
class TestWildcardSignal:
def test_connecting_to_every_signal_is_rejected(self) -> None:
def handler(**kwargs: Any) -> None:
pass
with pytest.raises(ValueError, match="every signal"):
SignalManager(object()).connect(handler, registry.Any)

2
tests/utils/engine.py
View File

@ -8,7 +8,6 @@ from urllib.parse import urlparse
import attr
from itemadapter import ItemAdapter
from pydispatch import dispatcher
from twisted.internet import defer
from scrapy import signals
@ -16,6 +15,7 @@ from scrapy.http import Headers, Request, Response
from scrapy.item import Field, Item
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Spider
from scrapy.utils import _signal_registry as dispatcher
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.signal import disconnect_all
from scrapy.utils.test import get_crawler

4
tox.ini
View File

@ -136,6 +136,7 @@ deps =
pytest==8.4.0
Protego==0.1.15
Twisted==21.7.0
blinker==1.7.0
cryptography==37.0.0
cssselect==0.9.1
httpx2==2.0.0
@ -157,7 +158,6 @@ commands =
basepython = {[min]basepython}
deps =
{[min]deps}
PyDispatcher==2.0.5
setenv =
{[min]setenv}
commands = {[min]commands}
@ -250,11 +250,11 @@ commands = {[testenv:pypy3]commands}
[testenv:min-pypy3]
basepython = pypy3.11
deps =
PyPyDispatcher==2.1.0
{[test-requirements]deps}
pytest==8.4.0
Protego==0.1.15
Twisted==21.7.0
blinker==1.7.0
cryptography==44.0.2
cssselect==0.9.1
itemadapter==0.1.0