mirror of https://github.com/scrapy/scrapy.git
Document that is_asyncio_available() cannot be used in extension init.
Extensions are created while settings are applied, before the asyncio event loop is running, so the previous example would fail or mislead.
This commit is contained in:
parent
06af687662
commit
a6d9dce685
|
|
@ -25,6 +25,16 @@ Backward-incompatible changes
|
|||
|
||||
(:gh:`6585`, :gh:`7731`)
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- Documented that :func:`scrapy.utils.asyncio.is_asyncio_available` and
|
||||
:func:`scrapy.utils.reactorless.is_reactorless` cannot be used from
|
||||
extension initialization, and updated the asyncio requirement example to
|
||||
check the :setting:`TWISTED_REACTOR` and
|
||||
:setting:`TWISTED_REACTOR_ENABLED` settings instead.
|
||||
(:gh:`7812`)
|
||||
|
||||
.. _release-2.17.0:
|
||||
|
||||
Scrapy 2.17.0 (2026-07-07)
|
||||
|
|
|
|||
|
|
@ -117,25 +117,42 @@ Enforcing asyncio as a requirement
|
|||
==================================
|
||||
|
||||
If you are writing a :ref:`component <topics-components>` that requires asyncio
|
||||
to work, use :func:`scrapy.utils.asyncio.is_asyncio_available` to
|
||||
:ref:`enforce it as a requirement <enforce-component-requirements>`. For
|
||||
example:
|
||||
to work, :ref:`enforce it as a requirement <enforce-component-requirements>`.
|
||||
|
||||
Do **not** call :func:`scrapy.utils.asyncio.is_asyncio_available` from an
|
||||
extension ``__init__`` or ``from_crawler``. Extensions are created while
|
||||
settings are applied, before the asyncio event loop is running, so that
|
||||
function cannot detect asyncio support there. The same applies to
|
||||
:func:`scrapy.utils.reactorless.is_reactorless`. Check the
|
||||
:setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED` settings
|
||||
instead:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy.utils.asyncio import is_asyncio_available
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
||||
_ASYNCIO_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
|
||||
|
||||
|
||||
class MyComponent:
|
||||
def __init__(self):
|
||||
if not is_asyncio_available():
|
||||
raise ValueError(
|
||||
f"{MyComponent.__qualname__} requires the asyncio support. "
|
||||
f"Make sure you have configured the asyncio reactor in the "
|
||||
f"TWISTED_REACTOR setting. See the asyncio documentation "
|
||||
f"of Scrapy for more information."
|
||||
def __init__(self, crawler):
|
||||
reactor_enabled = crawler.settings.getbool("TWISTED_REACTOR_ENABLED")
|
||||
reactor = crawler.settings["TWISTED_REACTOR"]
|
||||
if reactor_enabled and reactor != _ASYNCIO_REACTOR:
|
||||
raise NotConfigured(
|
||||
f"{type(self).__qualname__} requires asyncio support. "
|
||||
f"Set TWISTED_REACTOR to {_ASYNCIO_REACTOR!r} "
|
||||
f"or set TWISTED_REACTOR_ENABLED to False."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler)
|
||||
|
||||
Use :func:`scrapy.utils.asyncio.is_asyncio_available` from code that runs
|
||||
after the crawl has started, such as spider callbacks and later component
|
||||
methods.
|
||||
|
||||
.. autofunction:: scrapy.utils.asyncio.is_asyncio_available
|
||||
.. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed
|
||||
|
||||
|
|
|
|||
|
|
@ -65,9 +65,21 @@ def is_asyncio_available() -> bool:
|
|||
calling it from code such as spiders and Scrapy components, if Scrapy
|
||||
is run using one of the supported ways).
|
||||
|
||||
.. note:: Do not call this function from an extension ``__init__`` or
|
||||
``from_crawler``. Extensions are created while settings are applied,
|
||||
before the asyncio event loop is running, so this function cannot
|
||||
detect asyncio support there (it may raise :exc:`RuntimeError` or
|
||||
return a misleading result). Check the :setting:`TWISTED_REACTOR`
|
||||
and :setting:`TWISTED_REACTOR_ENABLED` settings instead. Use this
|
||||
function from code that runs after the crawl has started.
|
||||
|
||||
.. versionchanged:: 2.15.0
|
||||
This function now also returns ``True`` if there is a running asyncio
|
||||
loop, even if no Twisted reactor is installed.
|
||||
|
||||
.. versionchanged:: VERSION
|
||||
Documented that this function is not usable from extension
|
||||
initialization.
|
||||
"""
|
||||
|
||||
# Check if there is a running asyncio loop.
|
||||
|
|
|
|||
|
|
@ -20,7 +20,10 @@ def is_reactorless() -> bool:
|
|||
|
||||
As this checks the runtime state and not the setting itself, it can be
|
||||
wrong when executed very early, before the reactor and/or the asyncio event
|
||||
loop are initialized.
|
||||
loop are initialized. In particular, do not call it from an extension
|
||||
``__init__`` or ``from_crawler``; those run while settings are applied,
|
||||
before the loop is running. Check the :setting:`TWISTED_REACTOR_ENABLED`
|
||||
setting instead.
|
||||
|
||||
.. note:: As this function uses
|
||||
:func:`scrapy.utils.asyncio.is_asyncio_available()`, it has the same
|
||||
|
|
|
|||
Loading…
Reference in New Issue