From 96437be1b52602e8e1b8592ff1dfcc743cc74c70 Mon Sep 17 00:00:00 2001 From: Rishi-gdrx Date: Tue, 7 Jul 2026 12:44:37 +0530 Subject: [PATCH] Address PR review feedback on docstring migration. Move contributor notes back to source comments, remove deprecated STATSMAILER_RCPTS docs, and fix the ftp_local_filename cross-reference. Co-authored-by: Cursor --- docs/contributing.rst | 11 ----------- docs/topics/download-handlers.rst | 2 ++ docs/topics/exceptions.rst | 5 ----- docs/topics/leaks.rst | 7 ------- docs/topics/request-response.rst | 2 +- docs/topics/settings.rst | 12 ------------ docs/topics/signals.rst | 4 ---- docs/topics/spiders.rst | 5 ----- scrapy/exceptions.py | 3 +++ scrapy/settings/default_settings.py | 8 ++++++++ scrapy/signals.py | 3 +++ scrapy/utils/sitemap.py | 3 +++ scrapy/utils/trackref.py | 5 +++++ 13 files changed, 25 insertions(+), 45 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 68cd1672c..c868a0ac4 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -308,17 +308,6 @@ In any case, if something is covered in a docstring, use the documentation instead of duplicating the docstring in files within the ``docs/`` directory. -When adding a new setting to ``scrapy/settings/default_settings.py``: - -* add it in alphabetical order, with the exception that enabling flags and - other high-level settings for a group should come first in their group - and pairs like host/port and user/password should be in the usual order -* group similar settings without leaving blank lines -* add its documentation to :doc:`topics/settings` - -When adding a new exception or signal, document it in :doc:`topics/exceptions` -or :doc:`topics/signals`, respectively. - Documentation updates that cover new or modified features must use Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use ``VERSION`` as version, we will replace it with the actual version right before diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 310b494fe..174237806 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -378,6 +378,8 @@ This handler supports ``ftp://host/path`` FTP URIs. It's implemented using :mod:`twisted.protocols.ftp`. +.. _topics-ftp-request-meta: + FTP connection parameters can be passed using :attr:`Request.meta `: diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 4508d6e60..cc690d609 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -7,11 +7,6 @@ Exceptions .. module:: scrapy.exceptions :synopsis: Scrapy exceptions -.. note:: - - When adding a new exception to ``scrapy.exceptions``, document it on this - page. - .. _topics-exceptions-ref: Built-in Exceptions reference diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 32286ab23..0913a3310 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -182,13 +182,6 @@ scrapy.utils.trackref module Here are the functions available in the :mod:`~scrapy.utils.trackref` module. -.. note:: - - PyPy uses a tracing garbage collector, so objects may remain in - ``live_refs`` longer than expected, even after they go out of scope. If - deterministic behavior is required, you may need to explicitly trigger - garbage collection or call ``trackref.live_refs.clear()``. - .. class:: object_ref Inherit from this class if you want to track live diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 7c84df2f0..6aa0f5845 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -721,7 +721,7 @@ Those are: * :reqmeta:`download_timeout` * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * ``ftp_passive`` (See :setting:`FTP_PASSIVE_MODE` for more info) -* ``ftp_local_filename`` (See :ref:`topics-download-handlers` for more info) +* ``ftp_local_filename`` (See :ref:`topics-ftp-request-meta` for more info) * ``ftp_user`` (See :setting:`FTP_USER` for more info) * :reqmeta:`give_up_log_level` * :reqmeta:`handle_httpstatus_all` diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 005bedcc9..11251a92c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2056,18 +2056,6 @@ Example: .. note:: This is a :ref:`pre-crawler setting `. -.. setting:: STATSMAILER_RCPTS - -STATSMAILER_RCPTS ------------------ - -Default: ``[]`` - -A list of email addresses to receive a message with scraping stats when a -spider finishes. Used by the deprecated -:class:`~scrapy.extensions.statsmailer.StatsMailer` extension, which is enabled -when this setting is non-empty. - .. setting:: STATS_CLASS STATS_CLASS diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index d0ed7110a..d13733623 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -16,10 +16,6 @@ deliver the arguments that the handler receives. You can connect to signals (or send your own) through the :ref:`topics-api-signals`. -.. note:: - - When adding a new signal to ``scrapy.signals``, document it on this page. - Here is a simple example showing how you can catch signals and perform some action: .. code-block:: python diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 11bce19cd..506daf930 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -787,11 +787,6 @@ SitemapSpider SitemapSpider allows you to crawl a site by discovering the URLs using `Sitemaps`_. - .. note:: - - The :mod:`scrapy.utils.sitemap` module provides sitemap parsing support - for this spider. Its API is subject to change without notice. - It supports nested sitemaps and discovering sitemap urls from `robots.txt`_. diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 54a2cdbd6..f611c4cff 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -1,3 +1,6 @@ +# These exceptions are documented in docs/topics/exceptions.rst. Please don't add +# new exceptions here without documenting them there. + from __future__ import annotations from typing import TYPE_CHECKING, Any diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 76cc85f19..4c7a209b2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -1,3 +1,11 @@ +# Scrapy developers, if you add a setting here remember to: +# +# * add it in alphabetical order, with the exception that enabling flags and +# other high-level settings for a group should come first in their group +# and pairs like host/port and user/password should be in the usual order +# * group similar settings without leaving blank lines +# * add its documentation to docs/topics/settings.rst + import sys from importlib import import_module from pathlib import Path diff --git a/scrapy/signals.py b/scrapy/signals.py index 383ef0245..81c737883 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -1,3 +1,6 @@ +# These signals are documented in docs/topics/signals.rst. Please don't add new +# signals here without documenting them there. + engine_started = object() engine_stopped = object() scheduler_empty = object() diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 2edf2d5ef..11e8e98ef 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -1,3 +1,6 @@ +# The main purpose of this module is to provide support for the SitemapSpider, +# its API is subject to change without notice. + from __future__ import annotations import warnings diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 7822f99b7..8bb5c4fa3 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -1,3 +1,8 @@ +# PyPy uses a tracing garbage collector, so objects may remain in live_refs +# longer than expected, even after they go out of scope. If deterministic +# behavior is required, you may need to explicitly trigger garbage collection +# or call trackref.live_refs.clear(). + from __future__ import annotations from collections import defaultdict