Assorted docstring fixes (#7698)

* Smaller fixes.

* Add more code blocks in docstrings.

* Queue stuff.

* Response stuff.

* Round 2.

* Address feedback.

scrapy/addons.py

@@ -64,7 +64,7 @@ class AddonManager:
 
         :param settings: The :class:`~scrapy.settings.BaseSettings` object from \
             which to read the early add-on configuration
-        :type settings: :class:`~scrapy.settings.Settings`
+        :type settings: :class:`~scrapy.settings.BaseSettings`
         """
         for clspath in build_component_list(settings["ADDONS"]):
             addoncls = load_object(clspath)

scrapy/commands/__init__.py

@@ -73,7 +73,7 @@ class ScrapyCommand(ABC):
     def long_desc(self) -> str:
         """A long description of the command. Return short description when not
         available. It cannot contain newlines since contents will be formatted
-        by optparser which removes newlines and wraps text.
+        by argparse which removes newlines and wraps text.
         """
         return self.short_desc()
 

scrapy/core/downloader/contextfactory.py

@@ -47,8 +47,13 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
     instance.
 
     The purpose of this custom class is to provide a ``creatorForNetloc()``
-    method that returns a ``_ScrapyClientTLSOptions`` instance configured based
-    on TLS settings provided to the factory.
+    method that returns:
+
+    - a ``_ScrapyClientTLSOptions26`` or ``_ScrapyClientTLSOptions`` instance
+      configured based on TLS settings provided to the factory (when the
+      certificate verification is disabled);
+    - a result of ``optionsForClientTLS()`` called with those TLS settings
+      (when the certificate verification is enabled).
     """
 
     def __init__(

scrapy/core/downloader/handlers/ftp.py

@@ -2,9 +2,9 @@
 An asynchronous FTP file download handler for scrapy which somehow emulates an http response.
 
 FTP connection parameters are passed using the request meta field:
-- ftp_user (required)
-- ftp_password (required)
-- ftp_passive (by default, enabled) sets FTP connection passive mode
+- ftp_user (optional, falls back to FTP_USER)
+- ftp_password (optional, falls back to FTP_PASSWORD)
+- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode
 - ftp_local_filename
         - If not given, file data will come in the response.body, as a normal scrapy Response,
         which will imply that the entire file will be on memory.

scrapy/core/downloader/handlers/http11.py

@@ -104,7 +104,6 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
         self._disconnect_timeout: int = 1
 
     async def download_request(self, request: Request) -> Response:
-        """Return a deferred for the HTTP download"""
         if hasattr(self._crawler.spider, "download_maxsize"):  # pragma: no cover
             warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")
         if hasattr(self._crawler.spider, "download_warnsize"):  # pragma: no cover
@@ -283,7 +282,7 @@ def _tunnel_request_data(
 
 
 class _TunnelingAgent(Agent):
-    """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
+    """An agent that uses a ``_TunnelingTCP4ClientEndpoint`` to make HTTPS
     downloads. It may look strange that we have chosen to subclass Agent and not
     ProxyAgent but consider that after the tunnel is opened the proxy is
     transparent to the client; thus the agent should behave like there is no

scrapy/core/http2/agent.py

@@ -114,11 +114,7 @@ class H2ConnectionPool:
             d.errback(ResponseFailed(errors))
 
     def close_connections(self) -> None:
-        """Close all the HTTP/2 connections and remove them from pool
-
-        Returns:
-            Deferred that fires when all connections have been closed
-        """
+        """Close all the HTTP/2 connections and remove them from pool."""
         for conn in self._connections.values():
             assert conn.transport is not None  # typing
             conn.transport.abortConnection()

scrapy/core/http2/protocol.py

@@ -101,7 +101,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
                 uri is used to verify that incoming client requests have correct
                 base URL.
             settings -- Scrapy project settings
-            conn_lost_deferred -- Deferred fires with the reason: Failure to notify
+            conn_lost_deferred -- Deferred that fires with the list of underlying exceptions to notify
                 that connection was lost
             tls_verbose_logging -- Whether to log TLS details
         """
@@ -375,7 +375,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
     def _handle_events(self, events: list[Event]) -> None:
         """Private method which acts as a bridge between the events
-        received from the HTTP/2 data and IH2EventsHandler
+        received from the HTTP/2 data and the handlers in this class.
 
         Arguments:
             events -- A list of events that the remote peer triggered by sending data

scrapy/core/scheduler.py

@@ -131,10 +131,10 @@ class Scheduler(BaseScheduler):
     (:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by
     :attr:`~scrapy.http.Request.priority`.
 
-    By default, a single, memory-based priority queue is used for all requests.
-    When using :setting:`JOBDIR`, a disk-based priority queue is also created,
+    By default, memory-based priority queues are used for all requests.
+    When using :setting:`JOBDIR`, disk-based priority queues are also created,
     and only unserializable requests are stored in the memory-based priority
-    queue. For a given priority value, requests in memory take precedence over
+    queues. For a given priority value, requests in memory take precedence over
     requests in disk.
 
     Each priority queue stores requests in separate internal queues, one per
@@ -209,8 +209,8 @@ class Scheduler(BaseScheduler):
     -------------------------
 
     While pending requests are below the configured values of
-    :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
-    or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent
+    :setting:`CONCURRENT_REQUESTS` or
+    :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent
     concurrently.
 
     As a result, the first few requests of a crawl may not follow the desired
@@ -342,7 +342,7 @@ class Scheduler(BaseScheduler):
     def open(self, spider: Spider) -> Deferred[None] | None:
         """
         (1) initialize the memory queue
-        (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
+        (2) initialize the disk queue if the ``jobdir`` argument wasn't empty
         (3) return the result of the dupefilter's ``open`` method
         """
         self.spider: Spider = spider

scrapy/core/scraper.py

@@ -441,7 +441,7 @@ class Scraper:
         self, output: Any, response: Response | Failure
     ) -> Deferred[None]:
         """Process each Request/Item (given in the output parameter) returned
-        from the given spider.
+        from the spider.
 
         Items are sent to the item pipelines, requests are scheduled.
         """
@@ -451,7 +451,7 @@ class Scraper:
         self, output: Any, response: Response | Failure
     ) -> None:
         """Process each Request/Item (given in the output parameter) returned
-        from the given spider.
+        from the spider.
 
         Items are sent to the item pipelines, requests are scheduled.
         """

scrapy/crawler.py

@@ -531,8 +531,8 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
         """
         Run a crawler with the provided arguments.
 
-        It will call the given Crawler's :meth:`~Crawler.crawl` method, while
-        keeping track of it so it can be stopped later.
+        It will call the given Crawler's :meth:`~Crawler.crawl_async` method,
+        while keeping track of it so it can be stopped later.
 
         If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler`
         instance, this method will try to create one using this parameter as
@@ -773,7 +773,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):
         """
         This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
         size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS
-        resolver based on :setting:`DNSCACHE_ENABLED`.
+        resolver based on :setting:`TWISTED_DNS_RESOLVER`.
 
         If ``stop_after_crawl`` is True, the reactor will be stopped after all
         crawlers have finished, using :meth:`join`.
@@ -875,10 +875,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
 
         When using a reactor it adjusts its pool size to
         :setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based
-        on :setting:`DNSCACHE_ENABLED`.
+        on :setting:`TWISTED_DNS_RESOLVER`.
 
-        If ``stop_after_crawl`` is True, the reactor will be stopped after all
-        crawlers have finished, using :meth:`join`.
+        If ``stop_after_crawl`` is True, the reactor/event loop will be stopped
+        after all crawlers have finished, using :meth:`join`.
 
         :param bool stop_after_crawl: stop or not the reactor when all
             crawlers have finished

scrapy/downloadermiddlewares/httpcompression.py

@@ -60,7 +60,7 @@ else:
 
 
 class HttpCompressionMiddleware:
-    """This middleware allows compressed (gzip, deflate) traffic to be
+    """This middleware allows compressed (gzip, deflate etc.) traffic to be
     sent/received from websites"""
 
     def __init__(

scrapy/downloadermiddlewares/redirect.py

@@ -196,10 +196,7 @@ class BaseRedirectMiddleware:
 
 
 class RedirectMiddleware(BaseRedirectMiddleware):
-    """
-    Handle redirection of requests based on response status
-    and meta-refresh html tag.
-    """
+    """Handle redirection of requests based on response status."""
 
     @_warn_spider_arg
     def process_response(
@@ -251,6 +248,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
 
 class MetaRefreshMiddleware(BaseRedirectMiddleware):
+    """Handle redirection of requests based on meta-refresh html tag."""
+
     enabled_setting = "METAREFRESH_ENABLED"
 
     def __init__(self, settings: BaseSettings):

scrapy/downloadermiddlewares/retry.py

@@ -5,9 +5,6 @@ problems such as a connection timeout or HTTP 500 error.
 You can change the behaviour of this middleware by modifying the scraping settings:
 RETRY_TIMES - how many times to retry a failed page
 RETRY_HTTP_CODES - which HTTP response codes to retry
-
-Failed pages are collected on the scraping process and rescheduled at the end,
-once the spider has finished crawling all regular (non-failed) pages.
 """
 
 from __future__ import annotations
@@ -70,8 +67,9 @@ def get_retry_request(
     and :ref:`stats <topics-stats>`, and to provide extra logging context (see
     :func:`logging.debug`).
 
-    *reason* is a string or an :class:`Exception` object that indicates the
-    reason why the request needs to be retried. It is used to name retry stats.
+    *reason* is a string, an :class:`Exception` subclass or an
+    :class:`Exception` object that indicates the reason why the request needs
+    to be retried. It is used to name retry stats.
 
     *max_retry_times* is a number that determines the maximum number of times
     that *request* can be retried. If not specified or ``None``, the number is

scrapy/exceptions.py

@@ -115,7 +115,7 @@ class UsageError(Exception):
 
 class ScrapyDeprecationWarning(Warning):
     """Warning category for deprecated features, since the default
-    DeprecationWarning is silenced on Python 2.7+
+    :exc:`DeprecationWarning` is silenced.
     """
 
 

scrapy/extensions/logstats.py

@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
 
 class LogStats:
     """Log basic scraping stats periodically like:
-    * RPM - Requests per Minute
+    * RPM - Responses per Minute
     * IPM - Items per Minute
     """
 

scrapy/extensions/throttle.py

@@ -116,7 +116,7 @@ class AutoThrottle:
         # It works better with problematic sites.
         new_delay = max(target_delay, new_delay)
 
-        # Make sure self.mindelay <= new_delay <= self.max_delay
+        # Make sure self.mindelay <= new_delay <= self.maxdelay
         new_delay = min(max(self.mindelay, new_delay), self.maxdelay)
 
         # Dont adjust delay if response status != 200 and new delay is smaller

scrapy/http/cookies.py

@@ -136,9 +136,9 @@ class _DummyLock:
 
 
 class WrappedRequest:
-    """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class
-
-    see http://docs.python.org/library/urllib2.html#urllib2.Request
+    """Wraps a :class:`scrapy.Request` class with methods defined by
+    the :class:`urllib.request.Request` class to interact with
+    the :class:`http.cookiejar.CookieJar` class.
     """
 
     def __init__(self, request: Request):

scrapy/http/response/html.py

@@ -1,6 +1,7 @@
 """
-This module implements the HtmlResponse class which adds encoding
-discovering through HTML encoding declarations to the TextResponse class.
+This module implements the :class:`HtmlResponse` class which is used as a
+content type marker by :class:`~scrapy.selector.Selector` and can be used in
+``isinstance()`` checks.
 
 See documentation in docs/topics/request-response.rst
 """

scrapy/http/response/xml.py

@@ -1,6 +1,7 @@
 """
-This module implements the XmlResponse class which adds encoding
-discovering through XML encoding declarations to the TextResponse class.
+This module implements the :class:`XmlResponse` class which is used as a
+content type marker by :class:`~scrapy.selector.Selector` and can be used in
+``isinstance()`` checks.
 
 See documentation in docs/topics/request-response.rst
 """

scrapy/item.py

@@ -1,7 +1,7 @@
 """
 Scrapy Item
 
-See documentation in docs/topics/item.rst
+See documentation in docs/topics/items.rst
 """
 
 from __future__ import annotations

scrapy/link.py

@@ -9,7 +9,9 @@ its documentation in: docs/topics/link-extractors.rst
 class Link:
     """Link objects represent an extracted link by the LinkExtractor.
 
-    Using the anchor tag sample below to illustrate the parameters::
+    Using the anchor tag sample below to illustrate the parameters:
+
+    .. code-block:: html
 
             <a href="https://example.com/nofollow.html#foo" rel="nofollow">Dont follow this one</a>
 

scrapy/logformatter.py

@@ -58,18 +58,20 @@ class LogFormatter:
     logging an action the method must return ``None``.
 
     Here is an example on how to create a custom log formatter to lower the severity level of
-    the log message when an item is dropped from the pipeline::
+    the log message when an item is dropped from the pipeline:
 
-            class PoliteLogFormatter(logformatter.LogFormatter):
-                def dropped(self, item, exception, response, spider):
-                    return {
-                        'level': logging.INFO, # lowering the level from logging.WARNING
-                        'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
-                        'args': {
-                            'exception': exception,
-                            'item': item,
-                        }
-                    }
+    .. code-block:: python
+
+        class PoliteLogFormatter(logformatter.LogFormatter):
+            def dropped(self, item, exception, response, spider):
+                return {
+                    "level": logging.INFO,  # lowering the level from logging.WARNING
+                    "msg": "Dropped: %(exception)s" + os.linesep + "%(item)s",
+                    "args": {
+                        "exception": exception,
+                        "item": item,
+                    },
+                }
     """
 
     def crawled(

scrapy/mail.py

@@ -1,7 +1,5 @@
 """
 Mail sending helpers
-
-See documentation in docs/topics/email.rst
 """
 
 from __future__ import annotations

scrapy/pipelines/__init__.py

@@ -1,7 +1,7 @@
 """
 Item pipeline
 
-See documentation in docs/item-pipeline.rst
+See documentation in docs/topics/item-pipeline.rst
 """
 
 from __future__ import annotations

scrapy/pipelines/files.py

@@ -423,7 +423,7 @@ class FTPFilesStore:
 
 
 class FilesPipeline(MediaPipeline):
-    """Abstract pipeline that implement the file downloading
+    """Pipeline that implements file downloading.
 
     This pipeline tries to minimize network transfers and file processing,
     doing stat of the files and determining if file is new, up-to-date or

scrapy/pipelines/images.py

@@ -47,7 +47,7 @@ class ImageException(FileException):
 
 
 class ImagesPipeline(FilesPipeline):
-    """Abstract pipeline that implement the image thumbnail generation logic"""
+    """Pipeline that implements the handling logic specific to images."""
 
     MEDIA_NAME: str = "image"
 

scrapy/pqueues.py

@@ -92,9 +92,10 @@ class ScrapyPriorityQueue:
     -   The :data:`~scrapy.Request.priority` of the request.
 
     For each combination of the above seen, this class creates an instance of
-    *downstream_queue_cls* with *key* set to a subdirectory of the persistence
-    directory, named as the request priority (e.g. ``1``), with an ``s`` suffix
-    in case of a start request (e.g. ``1s``).
+    *downstream_queue_cls* (or *start_queue_cls* for start requests if it was
+    passed) with *key* set to a subdirectory of the persistence directory,
+    named as the negated request priority (e.g. ``-1``), with an ``s`` suffix
+    in case of a start request (e.g. ``-1s``).
     """
 
     @classmethod

scrapy/settings/__init__.py

@@ -454,9 +454,10 @@ class BaseSettings(MutableMapping[str, Any]):
         """
         Store a key/value attribute with a given priority.
 
-        Settings should be populated *before* configuring the Crawler object
-        (through the :meth:`~scrapy.crawler.Crawler.configure` method),
-        otherwise they won't have any effect.
+        Settings should be populated *before* the Crawler object applies them
+        (in the :meth:`~scrapy.crawler.Crawler.crawl_async` or
+        :meth:`~scrapy.crawler.Crawler.crawl` method), otherwise they won't
+        have any effect.
 
         :param name: the setting name
         :type name: str
@@ -613,7 +614,7 @@ class BaseSettings(MutableMapping[str, Any]):
         """
         Make a deep copy of current settings.
 
-        This method returns a new instance of the :class:`Settings` class,
+        This method returns a new instance of this class,
         populated with the same values and their priorities.
 
         Modifications to the new object won't be reflected on the original
@@ -658,7 +659,7 @@ class BaseSettings(MutableMapping[str, Any]):
         Make a copy of current settings and convert to a dict.
 
         This method returns a new dict populated with the same values
-        and their priorities as the current settings.
+        as the current settings.
 
         Modifications to the returned dict won't be reflected on the original
         settings.

scrapy/shell.py

@@ -75,7 +75,7 @@ if TYPE_CHECKING:
 # running event loop.
 #
 # Side note: it should be possible to remove _request_deferred() by using
-# engine.download_async() instead of engine.schedule(), losing the usual stuff
+# engine.download_async() instead of engine.crawl(), losing the usual stuff
 # like spider middlewares (none of which should be important).
 #
 # Other architecture problems:
@@ -188,7 +188,8 @@ class Shell:
     async def _schedule(self, request: Request, spider: Spider | None) -> Response:
         """Send the request to the engine, wait for the result.
 
-        Runs in the reactor thread.
+        Runs in the reactor thread when using the reactor, or in the asyncio
+        event loop thread otherwise.
         """
         if not self.spider:
             await self._open_spider(spider)

scrapy/spidermiddlewares/base.py

@@ -75,7 +75,7 @@ class BaseSpiderMiddleware:
     ) -> Request | None:
         """Return a processed request from the spider output.
 
-        This method is called with a single request from the start seeds or the
+        This method is called with a single request from ``start()`` or the
         spider output. It should return the same or a different request, or
         ``None`` to ignore it.
 
@@ -84,7 +84,7 @@ class BaseSpiderMiddleware:
 
         :param response: the response being processed
         :type response: :class:`~scrapy.http.Response` object or ``None`` for
-            start seeds
+            start requests
 
         :return: the processed request or ``None``
         """
@@ -93,7 +93,7 @@ class BaseSpiderMiddleware:
     def get_processed_item(self, item: Any, response: Response | None) -> Any:
         """Return a processed item from the spider output.
 
-        This method is called with a single item from the start seeds or the
+        This method is called with a single item from ``start()`` or the
         spider output. It should return the same or a different item, or
         ``None`` to ignore it.
 
@@ -102,7 +102,7 @@ class BaseSpiderMiddleware:
 
         :param response: the response being processed
         :type response: :class:`~scrapy.http.Response` object or ``None`` for
-            start seeds
+            start items
 
         :return: the processed item or ``None``
         """

scrapy/spidermiddlewares/httperror.py

@@ -28,7 +28,7 @@ logger = logging.getLogger(__name__)
 
 
 class HttpError(IgnoreRequest):
-    """A non-200 response was filtered"""
+    """A non-2xx response was filtered"""
 
     def __init__(self, response: Response, *args: Any, **kwargs: Any):
         self.response = response

scrapy/spidermiddlewares/referer.py

@@ -92,7 +92,7 @@ class ReferrerPolicy(ABC):
         )
 
     def origin(self, url: str) -> str | None:
-        """Return serialized origin (scheme, host, path) for a request or response URL."""
+        """Return serialized origin (scheme, host, port) for a request or response URL."""
         return self.strip_url(url, origin_only=True)
 
     def potentially_trustworthy(self, url: str) -> bool:

scrapy/spiders/feed.py

@@ -54,17 +54,21 @@ class XMLFeedSpider(Spider):
         return response
 
     def parse_node(self, response: Response, selector: Selector) -> Any:
-        """This method must be overridden with your custom spider functionality"""
+        """This method is called for the nodes matching the provided tag name
+        (itertag). Receives the response and an Selector for each node.
+
+        This method must return either an item, a request, or a list
+        containing any of them.
+
+        This method must be overridden with your custom spider functionality.
+        """
         if hasattr(self, "parse_item"):  # backward compatibility
             return self.parse_item(response, selector)
         raise NotImplementedError
 
     def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any:
         """This method is called for the nodes matching the provided tag name
-        (itertag). Receives the response and an Selector for each node.
-        Overriding this method is mandatory. Otherwise, you spider won't work.
-        This method must return either an item, a request, or a list
-        containing any of them.
+        (itertag). Receives the response and an iterable of Selectors.
         """
 
         for selector in nodes:
@@ -113,6 +117,9 @@ class CSVFeedSpider(Spider):
     It receives a CSV file in a response; iterates through each of its rows,
     and calls parse_row with a dict containing each field's data.
 
+    This spider also gives the opportunity to override adapt_response and
+    process_results methods for pre and post-processing purposes.
+
     You can set some options regarding the CSV file, such as the delimiter, quotechar
     and the file's headers.
     """
@@ -136,16 +143,14 @@ class CSVFeedSpider(Spider):
         return response
 
     def parse_row(self, response: Response, row: dict[str, str]) -> Any:
-        """This method must be overridden with your custom spider functionality"""
+        """Receives a response and a dict (representing each row) with a key for
+        each provided (or detected) header of the CSV file.
+
+        This method must be overridden with your custom spider functionality.
+        """
         raise NotImplementedError
 
     def parse_rows(self, response: Response) -> Any:
-        """Receives a response and a dict (representing each row) with a key for
-        each provided (or detected) header of the CSV file.  This spider also
-        gives the opportunity to override adapt_response and
-        process_results methods for pre and post-processing purposes.
-        """
-
         for row in csviter(
             response, self.delimiter, self.headers, quotechar=self.quotechar
         ):

scrapy/utils/datatypes.py

@@ -1,8 +1,6 @@
 """
 This module contains data types used by Scrapy which are not included in the
 Python Standard Library.
-
-This module must not depend on any module outside the Standard Library.
 """
 
 from __future__ import annotations

scrapy/utils/defer.py

@@ -103,7 +103,7 @@ async def _defer_sleep_async() -> None:
 def defer_result(result: Any) -> Deferred[Any]:  # pragma: no cover
     warnings.warn(
         "scrapy.utils.defer.defer_result() is deprecated, use"
-        " twisted.internet.defer.success() and twisted.internet.defer.fail(),"
+        " twisted.internet.defer.succeed() and twisted.internet.defer.fail(),"
         " plus an explicit sleep if needed, or explicit reactor.callLater().",
         category=ScrapyDeprecationWarning,
         stacklevel=2,
@@ -469,22 +469,22 @@ def _maybeDeferred_coro(
 def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
     """Return an :class:`asyncio.Future` object that wraps *d*.
 
-    This function requires
-    :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be
-    installed.
+    This function requires an installed asyncio reactor or a running asyncio
+    event loop, see :ref:`using-asyncio`.
 
-    When :ref:`using the asyncio reactor <install-asyncio>`, you cannot await
-    on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy
-    callables defined as coroutines <coroutine-support>`, you can only await on
-    ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects
-    allows you to wait on them::
+    In this state you cannot await on :class:`~twisted.internet.defer.Deferred`
+    objects from :ref:`Scrapy callables defined as coroutines
+    <coroutine-support>`, you can only await on ``Future`` objects. Wrapping
+    ``Deferred`` objects into ``Future`` objects allows you to wait on them:
+
+    .. code-block:: python
 
         class MySpider(Spider):
             ...
+
             async def parse(self, response):
-                additional_request = scrapy.Request('https://example.org/price')
-                deferred = self.crawler.engine.download(additional_request)
-                additional_response = await deferred_to_future(deferred)
+                deferred = some_dfd_helper()
+                result = await deferred_to_future(deferred)
 
     .. versionchanged:: 2.14
         This function no longer installs an asyncio loop if called before the
@@ -492,7 +492,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
         in this case.
     """
     if not is_asyncio_available():
-        raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.")
+        raise RuntimeError(
+            "deferred_to_future() requires an installed asyncio reactor"
+            " or a running asyncio event loop."
+        )
     return d.asFuture(asyncio.get_event_loop())
 
 
@@ -501,23 +504,26 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]:
     defined as a coroutine <coroutine-support>`.
 
     What you can await in Scrapy callables defined as coroutines depends on the
-    value of :setting:`TWISTED_REACTOR`:
+    value of :setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED`:
 
-    -   When :ref:`using the asyncio reactor <install-asyncio>`, you can only
-        await on :class:`asyncio.Future` objects.
+    -   When :ref:`using the asyncio reactor <install-asyncio>`, or :ref:`not
+        using a reactor at all <asyncio-without-reactor>`, you can only await
+        on :class:`asyncio.Future` objects.
 
-    -   When not using the asyncio reactor, you can only await on
-        :class:`~twisted.internet.defer.Deferred` objects.
+    -   When :ref:`using a non-asyncio reactor <disable-asyncio>`, you can only
+        await on :class:`~twisted.internet.defer.Deferred` objects.
 
-    If you want to write code that uses ``Deferred`` objects but works with any
-    reactor, use this function on all ``Deferred`` objects::
+    If you want to write code that uses ``Deferred`` objects but works in both
+    of these states, use this function on all ``Deferred`` objects:
+
+    .. code-block:: python
 
         class MySpider(Spider):
             ...
+
             async def parse(self, response):
-                additional_request = scrapy.Request('https://example.org/price')
-                deferred = self.crawler.engine.download(additional_request)
-                additional_response = await maybe_deferred_to_future(deferred)
+                deferred = some_dfd_helper()
+                result = await maybe_deferred_to_future(deferred)
     """
     if not is_asyncio_available():
         return d

scrapy/utils/deprecate.py

@@ -43,15 +43,18 @@ def create_deprecated_class(
     It can be used to rename a base class in a library. For example, if we
     have
 
-        class OldName(SomeClass):
-            # ...
+    .. code-block:: python
 
-    and we want to rename it to NewName, we can do the following::
+        class OldName(SomeClass): ...
 
-        class NewName(SomeClass):
-            # ...
+    and we want to rename it to NewName, we can do the following:
 
-        OldName = create_deprecated_class('OldName', NewName)
+    .. code-block:: python
+
+        class NewName(SomeClass): ...
+
+
+        OldName = create_deprecated_class("OldName", NewName)
 
     Then, if user class inherits from OldName, warning is issued. Also, if
     some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)``

scrapy/utils/log.py

@@ -248,8 +248,7 @@ def logformatter_adapter(
 ) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]:
     """
     Helper that takes the dictionary output from the methods in LogFormatter
-    and adapts it into a tuple of positional arguments for logger.log calls,
-    handling backward compatibility as well.
+    and adapts it into a tuple of positional arguments for logger.log calls.
     """
 
     level = logkws.get("level", logging.INFO)

scrapy/utils/response.py

@@ -88,7 +88,7 @@ def open_in_browser(
 
 
         def parse_details(self, response):
-            if "item name" not in response.body:
+            if "item name" not in response.text:
                 open_in_browser(response)
     """
     # circular imports

scrapy/utils/signal.py

@@ -39,7 +39,7 @@ def send_catch_log(
     *arguments: TypingAny,
     **named: TypingAny,
 ) -> list[tuple[TypingAny, TypingAny]]:
-    """Like ``pydispatcher.robust.sendRobust()`` but it also logs errors and returns
+    """Like ``pydispatch.robust.sendRobust()`` but it also logs errors and returns
     Failures instead of exceptions.
     """
     dont_log = named.pop("dont_log", ())
@@ -172,9 +172,8 @@ async def _send_catch_log_asyncio(
 
     Returns a coroutine that completes once all signal handlers have finished.
 
-    This function requires
-    :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be
-    installed.
+    This function requires an installed asyncio reactor or a running asyncio
+    event loop.
 
     .. versionadded:: 2.14
     """

scrapy/utils/trackref.py

@@ -4,9 +4,7 @@ references to live object instances.
 If you want live objects for a particular class to be tracked, you only have to
 subclass from object_ref (instead of object).
 
-About performance: This library has a minimal performance impact when enabled,
-and no performance penalty at all when disabled (as object_ref becomes just an
-alias to object in that case).
+This library has a minimal performance impact.
 
 .. note:: PyPy uses a tracing garbage collector, so objects may
     remain in the ``live_refs`` longer than expected, even after they

scrapy/utils/url.py

@@ -117,8 +117,8 @@ def strip_url(
     - ``strip_credentials`` removes "user:password@"
     - ``strip_default_port`` removes ":80" (resp. ":443", ":21")
       from http:// (resp. https://, ftp://) URLs
-    - ``origin_only`` replaces path component with "/", also dropping
-      query and fragment components ; it also strips credentials
+    - ``origin_only`` replaces the  path component with "/", also dropping
+      the query component; it also strips credentials
     - ``strip_fragment`` drops any #fragment component
     """