From e121040db0318a6a4059a01626f1c0941dc32b62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 30 Nov 2023 11:54:47 +0100
Subject: [PATCH] Document built-in spider close reasons

---
 docs/topics/api.rst   | 22 ++++++++++++++--------
 scrapy/core/engine.py | 31 ++++++++++++++++++++++++++++++-
 scrapy/crawler.py     |  2 ++
 3 files changed, 46 insertions(+), 9 deletions(-)

diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 175c877de..73b747e38 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -81,14 +81,7 @@ how you :ref:`configure the downloader middlewares
         For an introduction on extensions and a list of available extensions on
         Scrapy see :ref:`topics-extensions`.
 
-    .. attribute:: engine
-
-        The execution engine, which coordinates the core crawling logic
-        between the scheduler, downloader and spiders.
-
-        Some extension may want to access the Scrapy engine, to inspect  or 
-        modify the downloader and scheduler behaviour, although this is an
-        advanced use and this API is not yet stable.
+    .. autoattribute:: engine
 
     .. attribute:: spider
 
@@ -277,3 +270,16 @@ class (which they all inherit from).
 
         Close the given spider. After this is called, no more specific stats
         can be accessed or collected.
+
+
+.. _engine:
+
+ExecutionEngine API
+===================
+
+.. module:: scrapy.core.engine
+   :synopsis: Execution engine
+
+.. autoclass:: ExecutionEngine
+
+    .. automethod:: close_spider
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index dd1f56f8c..21393c482 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -82,6 +82,15 @@ class Slot:
 
 
 class ExecutionEngine:
+    """The execution engine manages all the core :ref:`components
+    <topics-components>`, such as the :ref:`scheduler <topics-scheduler>`, the
+    downloader, or the :ref:`spider <topics-spiders>`, at run time.
+
+    Some components access the engine through :attr:`Crawler.engine
+    <scrapy.crawler.Crawler.engine>` to access or modify other components, or
+    use core functionality such as closing the running spider.
+    """
+
     def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None:
         self.crawler: "Crawler" = crawler
         self.settings: Settings = crawler.settings
@@ -401,7 +410,27 @@ class ExecutionEngine:
             self.close_spider(self.spider, reason=ex.reason)
 
     def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
-        """Close (cancel) spider and clear all its outstanding requests"""
+        """Stop the crawl with the specified *reason* and clear all its
+        outstanding requests.
+
+        *reason* is an arbitrary string. Built-in Scrapy :ref:`components
+        <topics-components>` use the following reasons:
+
+        -   ``finished``: When the crawl finishes normally.
+
+        -   ``shutdown``: When stopping the crawl is requested, usually by the
+            user through a system signal.
+
+        -   ``cancelled``: When :exc:`~scrapy.exceptions.CloseSpider` is
+            raised, e.g. from a spider callback, without a custom *reason*.
+
+        -   ``closespider_errorcount``, ``closespider_pagecount``,
+            ``closespider_itemcount``, ``closespider_timeout_no_item``: See
+            :class:`~scrapy.extensions.closespider.CloseSpider`.
+
+        -   ``memusage_exceeded``: See
+            :class:`~scrapy.extensions.memusage.MemoryUsage`.
+        """
         if self.slot is None:
             raise RuntimeError("Engine slot not assigned")
 
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 1d3a11208..afd371e2b 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -85,6 +85,8 @@ class Crawler:
         self.logformatter: Optional[LogFormatter] = None
         self.request_fingerprinter: Optional[RequestFingerprinter] = None
         self.spider: Optional[Spider] = None
+
+        #: Running instance of :class:`~scrapy.core.engine.ExecutionEngine`.
         self.engine: Optional[ExecutionEngine] = None
 
     def _update_root_log_handler(self) -> None: