Add more docs.

docs/conf.py

@@ -164,6 +164,7 @@ scrapy_intersphinx_enable = [
     "form2request",
     "itemloaders",
     "parsel",
+    "platformdirs",
     "pytest",
     "pypug",
     "scrapy-lint",

docs/requirements.in

@@ -6,4 +6,4 @@ sphinx-notfound-page
 sphinx-reredirects
 sphinx-rtd-theme
 sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.11
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.12

docs/requirements.txt

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile -p 3.13 docs/requirements.in -o docs/requirements.txt
+#    uv pip compile -p 3.14 docs/requirements.in -o docs/requirements.txt
 alabaster==1.0.0
     # via sphinx
 annotated-types==0.7.0
@@ -36,6 +36,7 @@ docutils==0.22.4
     #   sphinx
     #   sphinx-markdown-builder
     #   sphinx-rtd-theme
+    #   sphinx-scrapy
 filelock==3.25.2
     # via tldextract
 h2==4.3.0
@@ -156,7 +157,7 @@ sphinx-rtd-theme==3.1.0
     # via
     #   -r docs/requirements.in
     #   sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@6f8e5e0bbd171a857da480f7188f2a205041cb60
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@3581b4148e62f34f0a835cdd06eee5a6e0f5f843
     # via -r docs/requirements.in
 sphinx-sitemap==2.9.0
     # via sphinx-scrapy

docs/topics/extensions.rst

@@ -503,3 +503,69 @@ Remote control extension
    :synopsis: Remote control extension
 
 .. autoclass:: RemoteControl
+
+.. setting:: REMOTE_CONTROL_ENABLED
+
+REMOTE_CONTROL_ENABLED
+""""""""""""""""""""""
+
+Default: ``True``
+
+Whether to enable the :class:`RemoteControl` extension.
+
+.. setting:: REMOTE_CONTROL_JOBS_DIR
+
+REMOTE_CONTROL_JOBS_DIR
+"""""""""""""""""""""""
+
+Default: ``None``
+
+The directory for storing :class:`RemoteControl` job files. When this is set to
+``None``, a ``scrapy/jobfiles`` subdirectory in
+:func:`platformdirs.user_state_dir` is used.
+
+As job files contain authentication tokens necessary to connect to Scrapy
+processes, this directory should not be exposed to untrusted environments.
+
+.. setting:: REMOTE_CONTROL_TIMEOUT_DEFAULT
+
+REMOTE_CONTROL_TIMEOUT_DEFAULT
+""""""""""""""""""""""""""""""
+
+Default: ``30.0``
+
+The default timeout in seconds for running a single code snippet sent to the
+:class:`RemoteControl` ``/execute`` endpoint. You can override it for a single
+request via the ``timeout_sec`` request field.
+
+.. setting:: REMOTE_CONTROL_TIMEOUT_MAX
+
+REMOTE_CONTROL_TIMEOUT_MAX
+""""""""""""""""""""""""""
+
+Default: ``600.0``
+
+The maximum allowed value for the ``timeout_sec`` field of
+:class:`RemoteControl` ``/execute`` endpoint requests. Higher values will be
+clamped to this value.
+
+.. setting:: REMOTE_CONTROL_OUTPUT_MAX_BYTES
+
+REMOTE_CONTROL_OUTPUT_MAX_BYTES
+"""""""""""""""""""""""""""""""
+
+Default: ``65536``
+
+The maximum size of the ``output`` field in responses of :class:`RemoteControl`
+``/execute`` endpoint requests. Longer ones will be truncated.
+
+.. setting:: REMOTE_CONTROL_TRACEBACK_MAX_BYTES
+
+REMOTE_CONTROL_TRACEBACK_MAX_BYTES
+""""""""""""""""""""""""""""""""""
+
+Default: ``16384``
+
+The maximum size of the ``traceback`` field in responses of
+:class:`RemoteControl` ``/execute`` endpoint requests. Longer ones will be
+truncated.

docs/topics/security.rst

@@ -223,6 +223,39 @@ More generally, if you crawl URLs from untrusted sources, consider validating
 their schemes (and, where applicable, their hosts) before scheduling requests,
 to avoid server-side request forgery (SSRF) and similar issues.
 
+.. _security-remote-control:
+
+Remote control server
+=====================
+
+Scrapy enables the remote control HTTP server
+(:class:`scrapy.extensions.remote_control.RemoteControl`) by default
+(:setting:`REMOTE_CONTROL_ENABLED`). Its purpose is to run arbitrary code
+inside the Scrapy process, so anyone who can connect to it can do that.
+
+The server listens on a random localhost port and requires a token for
+authentication. This token is stored in a job file (see
+:setting:`REMOTE_CONTROL_JOBS_DIR` for the location of these files), so you
+should protect these files from unauthorized access. On Linux and macOS systems
+Scrapy sets file system permissions for job files and the directory containing
+them to be accessible only by the owner.
+
+.. note::
+
+    The server doesn't use HTTPS so it's possible to sniff the traffic or
+    tamper with it, but it requires the attacker to get access to the loopback
+    traffic.
+
+If you do not use this feature, disable it entirely:
+
+.. code-block:: python
+
+    REMOTE_CONTROL_ENABLED = False
+
+* **Pro:** removes a local code-execution surface and one less listening port.
+
+* **Con:** you can no longer inspect and control a running crawler through it.
+
 .. _security-telnet:
 
 Telnet console