From 73369e6833db01c57bee105f2e3cac39ff2e0e04 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Aug 2026 17:57:15 +0500 Subject: [PATCH] Add more docs. --- docs/conf.py | 1 + docs/requirements.in | 2 +- docs/requirements.txt | 5 +-- docs/topics/extensions.rst | 66 ++++++++++++++++++++++++++++++++++++++ docs/topics/security.rst | 33 +++++++++++++++++++ 5 files changed, 104 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index ad55231bc..fd0fc0f98 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -164,6 +164,7 @@ scrapy_intersphinx_enable = [ "form2request", "itemloaders", "parsel", + "platformdirs", "pytest", "pypug", "scrapy-lint", diff --git a/docs/requirements.in b/docs/requirements.in index 2a4e57c25..49091f9e7 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -6,4 +6,4 @@ sphinx-notfound-page sphinx-reredirects sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.11 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.12 diff --git a/docs/requirements.txt b/docs/requirements.txt index f003357f2..0080c3200 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.13 docs/requirements.in -o docs/requirements.txt +# uv pip compile -p 3.14 docs/requirements.in -o docs/requirements.txt alabaster==1.0.0 # via sphinx annotated-types==0.7.0 @@ -36,6 +36,7 @@ docutils==0.22.4 # sphinx # sphinx-markdown-builder # sphinx-rtd-theme + # sphinx-scrapy filelock==3.25.2 # via tldextract h2==4.3.0 @@ -156,7 +157,7 @@ sphinx-rtd-theme==3.1.0 # via # -r docs/requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@6f8e5e0bbd171a857da480f7188f2a205041cb60 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@3581b4148e62f34f0a835cdd06eee5a6e0f5f843 # via -r docs/requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 1a589b8e5..1d04c7834 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -503,3 +503,69 @@ Remote control extension :synopsis: Remote control extension .. autoclass:: RemoteControl + +.. setting:: REMOTE_CONTROL_ENABLED + +REMOTE_CONTROL_ENABLED +"""""""""""""""""""""" + +Default: ``True`` + +Whether to enable the :class:`RemoteControl` extension. + +.. setting:: REMOTE_CONTROL_JOBS_DIR + +REMOTE_CONTROL_JOBS_DIR +""""""""""""""""""""""" + +Default: ``None`` + +The directory for storing :class:`RemoteControl` job files. When this is set to +``None``, a ``scrapy/jobfiles`` subdirectory in +:func:`platformdirs.user_state_dir` is used. + +As job files contain authentication tokens necessary to connect to Scrapy +processes, this directory should not be exposed to untrusted environments. + +.. setting:: REMOTE_CONTROL_TIMEOUT_DEFAULT + +REMOTE_CONTROL_TIMEOUT_DEFAULT +"""""""""""""""""""""""""""""" + +Default: ``30.0`` + +The default timeout in seconds for running a single code snippet sent to the +:class:`RemoteControl` ``/execute`` endpoint. You can override it for a single +request via the ``timeout_sec`` request field. + +.. setting:: REMOTE_CONTROL_TIMEOUT_MAX + +REMOTE_CONTROL_TIMEOUT_MAX +"""""""""""""""""""""""""" + +Default: ``600.0`` + +The maximum allowed value for the ``timeout_sec`` field of +:class:`RemoteControl` ``/execute`` endpoint requests. Higher values will be +clamped to this value. + +.. setting:: REMOTE_CONTROL_OUTPUT_MAX_BYTES + +REMOTE_CONTROL_OUTPUT_MAX_BYTES +""""""""""""""""""""""""""""""" + +Default: ``65536`` + +The maximum size of the ``output`` field in responses of :class:`RemoteControl` +``/execute`` endpoint requests. Longer ones will be truncated. + +.. setting:: REMOTE_CONTROL_TRACEBACK_MAX_BYTES + +REMOTE_CONTROL_TRACEBACK_MAX_BYTES +"""""""""""""""""""""""""""""""""" + +Default: ``16384`` + +The maximum size of the ``traceback`` field in responses of +:class:`RemoteControl` ``/execute`` endpoint requests. Longer ones will be +truncated. diff --git a/docs/topics/security.rst b/docs/topics/security.rst index 5348aae23..bb89aef31 100644 --- a/docs/topics/security.rst +++ b/docs/topics/security.rst @@ -223,6 +223,39 @@ More generally, if you crawl URLs from untrusted sources, consider validating their schemes (and, where applicable, their hosts) before scheduling requests, to avoid server-side request forgery (SSRF) and similar issues. +.. _security-remote-control: + +Remote control server +===================== + +Scrapy enables the remote control HTTP server +(:class:`scrapy.extensions.remote_control.RemoteControl`) by default +(:setting:`REMOTE_CONTROL_ENABLED`). Its purpose is to run arbitrary code +inside the Scrapy process, so anyone who can connect to it can do that. + +The server listens on a random localhost port and requires a token for +authentication. This token is stored in a job file (see +:setting:`REMOTE_CONTROL_JOBS_DIR` for the location of these files), so you +should protect these files from unauthorized access. On Linux and macOS systems +Scrapy sets file system permissions for job files and the directory containing +them to be accessible only by the owner. + +.. note:: + + The server doesn't use HTTPS so it's possible to sniff the traffic or + tamper with it, but it requires the attacker to get access to the loopback + traffic. + +If you do not use this feature, disable it entirely: + +.. code-block:: python + + REMOTE_CONTROL_ENABLED = False + +* **Pro:** removes a local code-execution surface and one less listening port. + +* **Con:** you can no longer inspect and control a running crawler through it. + .. _security-telnet: Telnet console