Smaller fixes.

scrapy/commands/__init__.py

@@ -73,7 +73,7 @@ class ScrapyCommand(ABC):
     def long_desc(self) -> str:
         """A long description of the command. Return short description when not
         available. It cannot contain newlines since contents will be formatted
-        by optparser which removes newlines and wraps text.
+        by argparse which removes newlines and wraps text.
         """
         return self.short_desc()
 

scrapy/core/downloader/contextfactory.py

@@ -47,8 +47,13 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
     instance.
 
     The purpose of this custom class is to provide a ``creatorForNetloc()``
-    method that returns a ``_ScrapyClientTLSOptions`` instance configured based
-    on TLS settings provided to the factory.
+    method that returns:
+
+    - a ``_ScrapyClientTLSOptions26`` or ``_ScrapyClientTLSOptions`` instance
+      configured based on TLS settings provided to the factory (when the
+      certificate verification is disabled);
+    - a result of ``optionsForClientTLS()`` called with those TLS settings
+      (when the certificate verification is enabled).
     """
 
     def __init__(

scrapy/core/downloader/handlers/http11.py

@@ -104,7 +104,6 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
         self._disconnect_timeout: int = 1
 
     async def download_request(self, request: Request) -> Response:
-        """Return a deferred for the HTTP download"""
         if hasattr(self._crawler.spider, "download_maxsize"):  # pragma: no cover
             warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")
         if hasattr(self._crawler.spider, "download_warnsize"):  # pragma: no cover
@@ -283,7 +282,7 @@ def _tunnel_request_data(
 
 
 class _TunnelingAgent(Agent):
-    """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
+    """An agent that uses a ``_TunnelingTCP4ClientEndpoint`` to make HTTPS
     downloads. It may look strange that we have chosen to subclass Agent and not
     ProxyAgent but consider that after the tunnel is opened the proxy is
     transparent to the client; thus the agent should behave like there is no

scrapy/core/http2/agent.py

@@ -114,11 +114,7 @@ class H2ConnectionPool:
             d.errback(ResponseFailed(errors))
 
     def close_connections(self) -> None:
-        """Close all the HTTP/2 connections and remove them from pool
-
-        Returns:
-            Deferred that fires when all connections have been closed
-        """
+        """Close all the HTTP/2 connections and remove them from pool."""
         for conn in self._connections.values():
             assert conn.transport is not None  # typing
             conn.transport.abortConnection()

scrapy/core/http2/protocol.py

@@ -101,7 +101,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
                 uri is used to verify that incoming client requests have correct
                 base URL.
             settings -- Scrapy project settings
-            conn_lost_deferred -- Deferred fires with the reason: Failure to notify
+            conn_lost_deferred -- Deferred that fires with the list of underlying exceptions to notify
                 that connection was lost
             tls_verbose_logging -- Whether to log TLS details
         """
@@ -375,7 +375,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
     def _handle_events(self, events: list[Event]) -> None:
         """Private method which acts as a bridge between the events
-        received from the HTTP/2 data and IH2EventsHandler
+        received from the HTTP/2 data and the handlers in this class.
 
         Arguments:
             events -- A list of events that the remote peer triggered by sending data

scrapy/core/scheduler.py

@@ -209,8 +209,8 @@ class Scheduler(BaseScheduler):
     -------------------------
 
     While pending requests are below the configured values of
-    :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
-    or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent
+    :setting:`CONCURRENT_REQUESTS` or
+    :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent
     concurrently.
 
     As a result, the first few requests of a crawl may not follow the desired

scrapy/core/scraper.py

@@ -441,7 +441,7 @@ class Scraper:
         self, output: Any, response: Response | Failure
     ) -> Deferred[None]:
         """Process each Request/Item (given in the output parameter) returned
-        from the given spider.
+        from the spider.
 
         Items are sent to the item pipelines, requests are scheduled.
         """
@@ -451,7 +451,7 @@ class Scraper:
         self, output: Any, response: Response | Failure
     ) -> None:
         """Process each Request/Item (given in the output parameter) returned
-        from the given spider.
+        from the spider.
 
         Items are sent to the item pipelines, requests are scheduled.
         """

scrapy/crawler.py

@@ -531,8 +531,8 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
         """
         Run a crawler with the provided arguments.
 
-        It will call the given Crawler's :meth:`~Crawler.crawl` method, while
-        keeping track of it so it can be stopped later.
+        It will call the given Crawler's :meth:`~Crawler.crawl_async` method,
+        while keeping track of it so it can be stopped later.
 
         If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler`
         instance, this method will try to create one using this parameter as
@@ -773,7 +773,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):
         """
         This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
         size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS
-        resolver based on :setting:`DNSCACHE_ENABLED`.
+        resolver based on :setting:`TWISTED_DNS_RESOLVER`.
 
         If ``stop_after_crawl`` is True, the reactor will be stopped after all
         crawlers have finished, using :meth:`join`.
@@ -875,10 +875,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
 
         When using a reactor it adjusts its pool size to
         :setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based
-        on :setting:`DNSCACHE_ENABLED`.
+        on :setting:`TWISTED_DNS_RESOLVER`.
 
-        If ``stop_after_crawl`` is True, the reactor will be stopped after all
-        crawlers have finished, using :meth:`join`.
+        If ``stop_after_crawl`` is True, the reactor/event loop will be stopped
+        after all crawlers have finished, using :meth:`join`.
 
         :param bool stop_after_crawl: stop or not the reactor when all
             crawlers have finished

scrapy/downloadermiddlewares/redirect.py

@@ -196,10 +196,7 @@ class BaseRedirectMiddleware:
 
 
 class RedirectMiddleware(BaseRedirectMiddleware):
-    """
-    Handle redirection of requests based on response status
-    and meta-refresh html tag.
-    """
+    """Handle redirection of requests based on response status."""
 
     @_warn_spider_arg
     def process_response(
@@ -251,6 +248,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
 
 class MetaRefreshMiddleware(BaseRedirectMiddleware):
+    """Handle redirection of requests based on meta-refresh html tag."""
+
     enabled_setting = "METAREFRESH_ENABLED"
 
     def __init__(self, settings: BaseSettings):

scrapy/exceptions.py

@@ -115,7 +115,7 @@ class UsageError(Exception):
 
 class ScrapyDeprecationWarning(Warning):
     """Warning category for deprecated features, since the default
-    DeprecationWarning is silenced on Python 2.7+
+    :exc:`DeprecationWarning` is silenced.
     """
 
 

scrapy/extensions/logstats.py

@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
 
 class LogStats:
     """Log basic scraping stats periodically like:
-    * RPM - Requests per Minute
+    * RPM - Responses per Minute
     * IPM - Items per Minute
     """
 

scrapy/http/cookies.py

@@ -136,9 +136,9 @@ class _DummyLock:
 
 
 class WrappedRequest:
-    """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class
-
-    see http://docs.python.org/library/urllib2.html#urllib2.Request
+    """Wraps a :class:`scrapy.Request` class with methods defined by
+    :class:`urllib.request.Request` class to interact with
+    :class:`http.cookiejar.CookieJar` class.
     """
 
     def __init__(self, request: Request):

scrapy/item.py

@@ -1,7 +1,7 @@
 """
 Scrapy Item
 
-See documentation in docs/topics/item.rst
+See documentation in docs/topics/items.rst
 """
 
 from __future__ import annotations

scrapy/mail.py

@@ -1,7 +1,5 @@
 """
 Mail sending helpers
-
-See documentation in docs/topics/email.rst
 """
 
 from __future__ import annotations

scrapy/pipelines/__init__.py

@@ -1,7 +1,7 @@
 """
 Item pipeline
 
-See documentation in docs/item-pipeline.rst
+See documentation in docs/topics/item-pipeline.rst
 """
 
 from __future__ import annotations

scrapy/pipelines/files.py

@@ -423,7 +423,7 @@ class FTPFilesStore:
 
 
 class FilesPipeline(MediaPipeline):
-    """Abstract pipeline that implement the file downloading
+    """Pipeline that implements the file downloading.
 
     This pipeline tries to minimize network transfers and file processing,
     doing stat of the files and determining if file is new, up-to-date or

scrapy/pipelines/images.py

@@ -47,7 +47,7 @@ class ImageException(FileException):
 
 
 class ImagesPipeline(FilesPipeline):
-    """Abstract pipeline that implement the image thumbnail generation logic"""
+    """Pipeline that implements the handling logic specific to images."""
 
     MEDIA_NAME: str = "image"
 

scrapy/settings/__init__.py

@@ -454,9 +454,10 @@ class BaseSettings(MutableMapping[str, Any]):
         """
         Store a key/value attribute with a given priority.
 
-        Settings should be populated *before* configuring the Crawler object
-        (through the :meth:`~scrapy.crawler.Crawler.configure` method),
-        otherwise they won't have any effect.
+        Settings should be populated *before* the Crawler object applies them
+        (in the :meth:`~scrapy.crawler.Crawler.crawl_async` or
+        :meth:`~scrapy.crawler.Crawler.crawl` method), otherwise they won't
+        have any effect.
 
         :param name: the setting name
         :type name: str
@@ -613,7 +614,7 @@ class BaseSettings(MutableMapping[str, Any]):
         """
         Make a deep copy of current settings.
 
-        This method returns a new instance of the :class:`Settings` class,
+        This method returns a new instance of this class,
         populated with the same values and their priorities.
 
         Modifications to the new object won't be reflected on the original
@@ -658,7 +659,7 @@ class BaseSettings(MutableMapping[str, Any]):
         Make a copy of current settings and convert to a dict.
 
         This method returns a new dict populated with the same values
-        and their priorities as the current settings.
+        as the current settings.
 
         Modifications to the returned dict won't be reflected on the original
         settings.

scrapy/shell.py

@@ -188,7 +188,8 @@ class Shell:
     async def _schedule(self, request: Request, spider: Spider | None) -> Response:
         """Send the request to the engine, wait for the result.
 
-        Runs in the reactor thread.
+        Runs in the reactor thread when using the reactor, or in the asyncio
+        event loop thread otherwise.
         """
         if not self.spider:
             await self._open_spider(spider)

scrapy/spidermiddlewares/base.py

@@ -75,7 +75,7 @@ class BaseSpiderMiddleware:
     ) -> Request | None:
         """Return a processed request from the spider output.
 
-        This method is called with a single request from the start seeds or the
+        This method is called with a single request from ``start()`` or the
         spider output. It should return the same or a different request, or
         ``None`` to ignore it.
 
@@ -84,7 +84,7 @@ class BaseSpiderMiddleware:
 
         :param response: the response being processed
         :type response: :class:`~scrapy.http.Response` object or ``None`` for
-            start seeds
+            start requests
 
         :return: the processed request or ``None``
         """
@@ -93,7 +93,7 @@ class BaseSpiderMiddleware:
     def get_processed_item(self, item: Any, response: Response | None) -> Any:
         """Return a processed item from the spider output.
 
-        This method is called with a single item from the start seeds or the
+        This method is called with a single item from ``start()`` or the
         spider output. It should return the same or a different item, or
         ``None`` to ignore it.
 
@@ -102,7 +102,7 @@ class BaseSpiderMiddleware:
 
         :param response: the response being processed
         :type response: :class:`~scrapy.http.Response` object or ``None`` for
-            start seeds
+            start items
 
         :return: the processed item or ``None``
         """

scrapy/spidermiddlewares/referer.py

@@ -92,7 +92,7 @@ class ReferrerPolicy(ABC):
         )
 
     def origin(self, url: str) -> str | None:
-        """Return serialized origin (scheme, host, path) for a request or response URL."""
+        """Return serialized origin (scheme, host, port) for a request or response URL."""
         return self.strip_url(url, origin_only=True)
 
     def potentially_trustworthy(self, url: str) -> bool:

scrapy/spiders/feed.py

@@ -54,17 +54,21 @@ class XMLFeedSpider(Spider):
         return response
 
     def parse_node(self, response: Response, selector: Selector) -> Any:
-        """This method must be overridden with your custom spider functionality"""
+        """This method is called for the nodes matching the provided tag name
+        (itertag). Receives the response and an Selector for each node.
+
+        This method must return either an item, a request, or a list
+        containing any of them.
+
+        This method must be overridden with your custom spider functionality.
+        """
         if hasattr(self, "parse_item"):  # backward compatibility
             return self.parse_item(response, selector)
         raise NotImplementedError
 
     def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any:
         """This method is called for the nodes matching the provided tag name
-        (itertag). Receives the response and an Selector for each node.
-        Overriding this method is mandatory. Otherwise, you spider won't work.
-        This method must return either an item, a request, or a list
-        containing any of them.
+        (itertag). Receives the response and an iterable of Selectors.
         """
 
         for selector in nodes:
@@ -113,6 +117,9 @@ class CSVFeedSpider(Spider):
     It receives a CSV file in a response; iterates through each of its rows,
     and calls parse_row with a dict containing each field's data.
 
+    This spider also gives the opportunity to override adapt_response and
+    process_results methods for pre and post-processing purposes.
+
     You can set some options regarding the CSV file, such as the delimiter, quotechar
     and the file's headers.
     """
@@ -136,16 +143,14 @@ class CSVFeedSpider(Spider):
         return response
 
     def parse_row(self, response: Response, row: dict[str, str]) -> Any:
-        """This method must be overridden with your custom spider functionality"""
+        """Receives a response and a dict (representing each row) with a key for
+        each provided (or detected) header of the CSV file.
+
+        This method must be overridden with your custom spider functionality.
+        """
         raise NotImplementedError
 
     def parse_rows(self, response: Response) -> Any:
-        """Receives a response and a dict (representing each row) with a key for
-        each provided (or detected) header of the CSV file.  This spider also
-        gives the opportunity to override adapt_response and
-        process_results methods for pre and post-processing purposes.
-        """
-
         for row in csviter(
             response, self.delimiter, self.headers, quotechar=self.quotechar
         ):

scrapy/utils/datatypes.py

@@ -1,8 +1,6 @@
 """
 This module contains data types used by Scrapy which are not included in the
 Python Standard Library.
-
-This module must not depend on any module outside the Standard Library.
 """
 
 from __future__ import annotations

scrapy/utils/defer.py

@@ -103,7 +103,7 @@ async def _defer_sleep_async() -> None:
 def defer_result(result: Any) -> Deferred[Any]:  # pragma: no cover
     warnings.warn(
         "scrapy.utils.defer.defer_result() is deprecated, use"
-        " twisted.internet.defer.success() and twisted.internet.defer.fail(),"
+        " twisted.internet.defer.succeed() and twisted.internet.defer.fail(),"
         " plus an explicit sleep if needed, or explicit reactor.callLater().",
         category=ScrapyDeprecationWarning,
         stacklevel=2,
@@ -469,22 +469,22 @@ def _maybeDeferred_coro(
 def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
     """Return an :class:`asyncio.Future` object that wraps *d*.
 
-    This function requires
-    :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be
-    installed.
+    This function requires an installed asyncio reactor or a running asyncio
+    event loop, see :ref:`using-asyncio`.
 
-    When :ref:`using the asyncio reactor <install-asyncio>`, you cannot await
-    on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy
-    callables defined as coroutines <coroutine-support>`, you can only await on
-    ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects
-    allows you to wait on them::
+    In this state you cannot await on :class:`~twisted.internet.defer.Deferred`
+    objects from :ref:`Scrapy callables defined as coroutines
+    <coroutine-support>`, you can only await on ``Future`` objects. Wrapping
+    ``Deferred`` objects into ``Future`` objects allows you to wait on them:
+
+    .. code-block:: python
 
         class MySpider(Spider):
             ...
+
             async def parse(self, response):
-                additional_request = scrapy.Request('https://example.org/price')
-                deferred = self.crawler.engine.download(additional_request)
-                additional_response = await deferred_to_future(deferred)
+                deferred = some_dfd_helper()
+                result = await deferred_to_future(deferred)
 
     .. versionchanged:: 2.14
         This function no longer installs an asyncio loop if called before the
@@ -492,7 +492,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
         in this case.
     """
     if not is_asyncio_available():
-        raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.")
+        raise RuntimeError(
+            "deferred_to_future() requires an installed asyncio reactor"
+            " or a running asyncio event loop."
+        )
     return d.asFuture(asyncio.get_event_loop())
 
 
@@ -501,23 +504,26 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]:
     defined as a coroutine <coroutine-support>`.
 
     What you can await in Scrapy callables defined as coroutines depends on the
-    value of :setting:`TWISTED_REACTOR`:
+    value of :setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED`:
 
-    -   When :ref:`using the asyncio reactor <install-asyncio>`, you can only
-        await on :class:`asyncio.Future` objects.
+    -   When :ref:`using the asyncio reactor <install-asyncio>`, or :ref:`not
+        using a reactor at all <asyncio-without-reactor>`, you can only await
+        on :class:`asyncio.Future` objects.
 
-    -   When not using the asyncio reactor, you can only await on
-        :class:`~twisted.internet.defer.Deferred` objects.
+    -   When :ref:`using a non-asyncio reactor <disable-asyncio>`, you can only
+        await on :class:`~twisted.internet.defer.Deferred` objects.
 
-    If you want to write code that uses ``Deferred`` objects but works with any
-    reactor, use this function on all ``Deferred`` objects::
+    If you want to write code that uses ``Deferred`` objects but works in both
+    of these states, use this function on all ``Deferred`` objects:
+
+    .. code-block:: python
 
         class MySpider(Spider):
             ...
+
             async def parse(self, response):
-                additional_request = scrapy.Request('https://example.org/price')
-                deferred = self.crawler.engine.download(additional_request)
-                additional_response = await maybe_deferred_to_future(deferred)
+                deferred = some_dfd_helper()
+                result = await deferred_to_future(deferred)
     """
     if not is_asyncio_available():
         return d

scrapy/utils/log.py

@@ -248,8 +248,7 @@ def logformatter_adapter(
 ) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]:
     """
     Helper that takes the dictionary output from the methods in LogFormatter
-    and adapts it into a tuple of positional arguments for logger.log calls,
-    handling backward compatibility as well.
+    and adapts it into a tuple of positional arguments for logger.log calls.
     """
 
     level = logkws.get("level", logging.INFO)

scrapy/utils/response.py

@@ -88,7 +88,7 @@ def open_in_browser(
 
 
         def parse_details(self, response):
-            if "item name" not in response.body:
+            if "item name" not in response.text:
                 open_in_browser(response)
     """
     # circular imports

scrapy/utils/signal.py

@@ -39,7 +39,7 @@ def send_catch_log(
     *arguments: TypingAny,
     **named: TypingAny,
 ) -> list[tuple[TypingAny, TypingAny]]:
-    """Like ``pydispatcher.robust.sendRobust()`` but it also logs errors and returns
+    """Like ``pydispatch.robust.sendRobust()`` but it also logs errors and returns
     Failures instead of exceptions.
     """
     dont_log = named.pop("dont_log", ())
@@ -172,9 +172,7 @@ async def _send_catch_log_asyncio(
 
     Returns a coroutine that completes once all signal handlers have finished.
 
-    This function requires
-    :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be
-    installed.
+    This function an installed asyncio reactor or a running asyncio event loop.
 
     .. versionadded:: 2.14
     """

scrapy/utils/trackref.py

@@ -4,9 +4,7 @@ references to live object instances.
 If you want live objects for a particular class to be tracked, you only have to
 subclass from object_ref (instead of object).
 
-About performance: This library has a minimal performance impact when enabled,
-and no performance penalty at all when disabled (as object_ref becomes just an
-alias to object in that case).
+This library has a minimal performance impact.
 
 .. note:: PyPy uses a tracing garbage collector, so objects may
     remain in the ``live_refs`` longer than expected, even after they

scrapy/utils/url.py

@@ -118,7 +118,7 @@ def strip_url(
     - ``strip_default_port`` removes ":80" (resp. ":443", ":21")
       from http:// (resp. https://, ftp://) URLs
     - ``origin_only`` replaces path component with "/", also dropping
-      query and fragment components ; it also strips credentials
+      query component; it also strips credentials
     - ``strip_fragment`` drops any #fragment component
     """