From 224b72b5a4835585858e44b2aa8a7c9a3c4beceb Mon Sep 17 00:00:00 2001 From: Rishi-gdrx Date: Sat, 4 Jul 2026 12:52:58 +0530 Subject: [PATCH 1/6] Remove duplicate module docstrings from exporters, extensions, and http. Fixes #7699 Co-authored-by: Cursor --- scrapy/exporters.py | 4 ---- scrapy/extensions/closespider.py | 6 ------ scrapy/extensions/corestats.py | 4 ---- scrapy/extensions/debug.py | 6 ------ scrapy/extensions/feedexport.py | 6 ------ scrapy/extensions/memdebug.py | 6 ------ scrapy/extensions/memusage.py | 6 ------ scrapy/extensions/postprocessing.py | 4 ---- scrapy/extensions/telnet.py | 6 ------ scrapy/http/__init__.py | 7 ------- scrapy/http/request/__init__.py | 7 ------- scrapy/http/request/form.py | 7 ------- scrapy/http/request/json_request.py | 7 ------- scrapy/http/request/rpc.py | 7 ------- scrapy/http/response/__init__.py | 7 ------- scrapy/http/response/html.py | 8 -------- scrapy/http/response/json.py | 7 ------- scrapy/http/response/text.py | 7 ------- scrapy/http/response/xml.py | 8 -------- 19 files changed, 120 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c3540d724..25f9d6a60 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -1,7 +1,3 @@ -""" -Item Exporters are used to export/serialize items into different formats. -""" - from __future__ import annotations import csv diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index a4362b182..1c007c7da 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -1,9 +1,3 @@ -"""CloseSpider is an extension that forces spiders to be closed after certain -conditions are met. - -See documentation in docs/topics/extensions.rst -""" - from __future__ import annotations import logging diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 6a5e55992..14d13e053 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -1,7 +1,3 @@ -""" -Extension for collecting core stats like items scraped and start/finish times -""" - from __future__ import annotations from datetime import datetime, timezone diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 5def7509e..67bcfd1ea 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -1,9 +1,3 @@ -""" -Extensions for debugging Scrapy - -See documentation in docs/topics/extensions.rst -""" - from __future__ import annotations import contextlib diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fa708a2e1..90199932e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -1,9 +1,3 @@ -""" -Feed Exports extension - -See documentation in docs/topics/feed-exports.rst -""" - from __future__ import annotations import asyncio diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 1fde6b296..ff4ef2703 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -1,9 +1,3 @@ -""" -MemoryDebugger extension - -See documentation in docs/topics/extensions.rst -""" - from __future__ import annotations import gc diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 1444c8941..0f58611e3 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -1,9 +1,3 @@ -""" -MemoryUsage extension - -See documentation in docs/topics/extensions.rst -""" - from __future__ import annotations import logging diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 5828ae52e..b108bee8c 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -1,7 +1,3 @@ -""" -Extension for processing data before they are exported to feeds. -""" - from bz2 import BZ2File from gzip import GzipFile from io import IOBase diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 3be24c53f..4b43c8338 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -1,9 +1,3 @@ -""" -Scrapy Telnet Console extension - -See documentation in docs/topics/telnetconsole.rst -""" - from __future__ import annotations import binascii diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index 0e5c2b53b..19340b6b6 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -1,10 +1,3 @@ -""" -Module containing all HTTP related classes - -Use this module (instead of the more specific ones) when importing Headers, -Request and Response outside this module. -""" - from scrapy.http.headers import Headers from scrapy.http.request import Request from scrapy.http.request.form import FormRequest diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 73c2e7dd4..acf1c57c6 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -1,10 +1,3 @@ -""" -This module implements the Request class which is used to represent HTTP -requests in Scrapy. - -See documentation in docs/topics/request-response.rst -""" - from __future__ import annotations import inspect diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index f1a8dbf3b..a97cfd2de 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -1,10 +1,3 @@ -""" -This module implements the FormRequest class which is a more convenient class -(than Request) to generate Requests based on form data. - -See documentation in docs/topics/request-response.rst -""" - from __future__ import annotations from collections.abc import Iterable diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 13fee06d8..97d877702 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -1,10 +1,3 @@ -""" -This module implements the JsonRequest class which is a more convenient class -(than Request) to generate JSON Requests. - -See documentation in docs/topics/request-response.rst -""" - from __future__ import annotations import copy diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index c68c8aabd..d0cededc5 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -1,10 +1,3 @@ -""" -This module implements the XmlRpcRequest class which is a more convenient class -(that Request) to generate xml-rpc requests. - -See documentation in docs/topics/request-response.rst -""" - from __future__ import annotations import xmlrpc.client as xmlrpclib diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 09b1c8b32..5690b3010 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -1,10 +1,3 @@ -""" -This module implements the Response class which is used to represent HTTP -responses in Scrapy. - -See documentation in docs/topics/request-response.rst -""" - from __future__ import annotations from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index 6d3a9ee12..e62846764 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -1,11 +1,3 @@ -""" -This module implements the :class:`HtmlResponse` class which is used as a -content type marker by :class:`~scrapy.selector.Selector` and can be used in -``isinstance()`` checks. - -See documentation in docs/topics/request-response.rst -""" - from scrapy.http.response.text import TextResponse diff --git a/scrapy/http/response/json.py b/scrapy/http/response/json.py index 0428dde6e..74b0ed0c9 100644 --- a/scrapy/http/response/json.py +++ b/scrapy/http/response/json.py @@ -1,10 +1,3 @@ -""" -This module implements the JsonResponse class that is used when the response -has a JSON MIME type in its Content-Type header. - -See documentation in docs/topics/request-response.rst -""" - from scrapy.http.response.text import TextResponse diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6876e35e8..c277d2412 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -1,10 +1,3 @@ -""" -This module implements the TextResponse class which adds encoding handling and -discovering (through HTTP headers) to base Response class. - -See documentation in docs/topics/request-response.rst -""" - from __future__ import annotations import json diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index 847fb2b3c..64496b8ca 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -1,11 +1,3 @@ -""" -This module implements the :class:`XmlResponse` class which is used as a -content type marker by :class:`~scrapy.selector.Selector` and can be used in -``isinstance()`` checks. - -See documentation in docs/topics/request-response.rst -""" - from scrapy.http.response.text import TextResponse From 751b384b87642d04aafdaee745dad6fea66ca7c1 Mon Sep 17 00:00:00 2001 From: Rishi-gdrx Date: Sat, 4 Jul 2026 12:54:53 +0530 Subject: [PATCH 2/6] Remove duplicate module docstrings from middleware modules. Fixes #7699 Co-authored-by: Cursor --- scrapy/core/downloader/middleware.py | 6 ------ scrapy/core/spidermw.py | 6 ------ scrapy/downloadermiddlewares/defaultheaders.py | 6 ------ scrapy/downloadermiddlewares/downloadtimeout.py | 6 ------ scrapy/downloadermiddlewares/httpauth.py | 6 ------ scrapy/downloadermiddlewares/retry.py | 9 --------- scrapy/downloadermiddlewares/robotstxt.py | 6 ------ scrapy/downloadermiddlewares/useragent.py | 2 -- scrapy/spidermiddlewares/depth.py | 6 ------ scrapy/spidermiddlewares/httperror.py | 6 ------ scrapy/spidermiddlewares/referer.py | 5 ----- scrapy/spidermiddlewares/urllength.py | 6 ------ 12 files changed, 70 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index ab74e22a4..2f811cac0 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -1,9 +1,3 @@ -""" -Downloader Middleware manager - -See documentation in docs/topics/downloader-middleware.rst -""" - from __future__ import annotations import warnings diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index fbc6f2530..bf31cb454 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -1,9 +1,3 @@ -""" -Spider Middleware manager - -See documentation in docs/topics/spider-middleware.rst -""" - from __future__ import annotations import logging diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 7c940a078..5aa8f37d7 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -1,9 +1,3 @@ -""" -DefaultHeaders downloader middleware - -See documentation in docs/topics/downloader-middleware.rst -""" - from __future__ import annotations from typing import TYPE_CHECKING diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index bccfb230c..79d0aebdf 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -1,9 +1,3 @@ -""" -Download timeout middleware - -See documentation in docs/topics/downloader-middleware.rst -""" - from __future__ import annotations from typing import TYPE_CHECKING diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index b09abdadd..2eaa9e532 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -1,9 +1,3 @@ -""" -HTTP basic auth downloader middleware - -See documentation in docs/topics/downloader-middleware.rst -""" - from __future__ import annotations import warnings diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index a0a0b60a2..6f2fe63e4 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -1,12 +1,3 @@ -""" -An extension to retry failed requests that are potentially caused by temporary -problems such as a connection timeout or HTTP 500 error. - -You can change the behaviour of this middleware by modifying the scraping settings: -RETRY_TIMES - how many times to retry a failed page -RETRY_HTTP_CODES - which HTTP response codes to retry -""" - from __future__ import annotations from logging import Logger, getLevelName, getLogger diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7d0c17884..4b125c00a 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -1,9 +1,3 @@ -""" -This is a middleware to respect robots.txt policies. To activate it you must -enable this middleware and enable the ROBOTSTXT_OBEY setting. - -""" - from __future__ import annotations import logging diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 61f84b518..a7438fe76 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -1,5 +1,3 @@ -"""Set User-Agent header per spider or use a default value from settings""" - from __future__ import annotations from typing import TYPE_CHECKING diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 9c387530c..9179c97de 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -1,9 +1,3 @@ -""" -Depth Spider Middleware - -See documentation in docs/topics/spider-middleware.rst -""" - from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 156b73e7e..009704833 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -1,9 +1,3 @@ -""" -HttpError Spider Middleware - -See documentation in docs/topics/spider-middleware.rst -""" - from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 1305874d5..42b036a95 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -1,8 +1,3 @@ -""" -RefererMiddleware: populates Request referer field, based on the Response which -originated it. -""" - from __future__ import annotations import warnings diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index f325ce7a0..a938cd69e 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -1,9 +1,3 @@ -""" -Url Length Spider Middleware - -See documentation in docs/topics/spider-middleware.rst -""" - from __future__ import annotations import logging From cc9006a56b669f0fd314cc9d1c2448c4107a2385 Mon Sep 17 00:00:00 2001 From: Rishi-gdrx Date: Sat, 4 Jul 2026 12:57:22 +0530 Subject: [PATCH 3/6] Remove duplicate module docstrings from spiders, items, loaders, shell, and pipelines. Fixes #7699 Co-authored-by: Cursor --- scrapy/commands/shell.py | 6 ------ scrapy/extension.py | 6 ------ scrapy/item.py | 6 ------ scrapy/loader/__init__.py | 6 ------ scrapy/pipelines/__init__.py | 6 ------ scrapy/pipelines/files.py | 6 ------ scrapy/pipelines/images.py | 6 ------ scrapy/shell.py | 6 ------ scrapy/spiders/__init__.py | 6 ------ scrapy/spiders/crawl.py | 7 ------- scrapy/spiders/feed.py | 7 ------- 11 files changed, 68 deletions(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 19138ffd0..1e57b4a8a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -1,9 +1,3 @@ -""" -Scrapy Shell - -See documentation in docs/topics/shell.rst -""" - from __future__ import annotations import asyncio diff --git a/scrapy/extension.py b/scrapy/extension.py index 05cce326b..768804b7f 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -1,9 +1,3 @@ -""" -The Extension Manager - -See documentation in docs/topics/extensions.rst -""" - from __future__ import annotations from typing import TYPE_CHECKING, Any diff --git a/scrapy/item.py b/scrapy/item.py index 4d99ea79d..79333a7d1 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -1,9 +1,3 @@ -""" -Scrapy Item - -See documentation in docs/topics/items.rst -""" - from __future__ import annotations from abc import ABCMeta diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 8ee7dc16c..7ac324fd0 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -1,9 +1,3 @@ -""" -Item Loader - -See documentation in docs/topics/loaders.rst -""" - from __future__ import annotations from typing import TYPE_CHECKING, Any diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 383c461e6..9e8bc4df3 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -1,9 +1,3 @@ -""" -Item pipeline - -See documentation in docs/topics/item-pipeline.rst -""" - from __future__ import annotations import asyncio diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 44c422430..1763bda42 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -1,9 +1,3 @@ -""" -Files Pipeline - -See documentation in topics/media-pipeline.rst -""" - from __future__ import annotations import base64 diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 79b6c4f27..cd2d5d43e 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -1,9 +1,3 @@ -""" -Images Pipeline - -See documentation in topics/media-pipeline.rst -""" - from __future__ import annotations import functools diff --git a/scrapy/shell.py b/scrapy/shell.py index dfea00c46..1856787c4 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -1,9 +1,3 @@ -"""Scrapy Shell - -See documentation in docs/topics/shell.rst - -""" - from __future__ import annotations import asyncio diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index de527d04f..52422ce62 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -1,9 +1,3 @@ -""" -Base class for Scrapy spiders - -See documentation in docs/topics/spiders.rst -""" - from __future__ import annotations import logging diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 373c0b8b5..19c054035 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -1,10 +1,3 @@ -""" -This modules implements the CrawlSpider which is the recommended spider to use -for scraping typical websites that requires crawling pages. - -See documentation in docs/topics/spiders.rst -""" - from __future__ import annotations import copy diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 925f31ede..b7d582092 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -1,10 +1,3 @@ -""" -This module implements the XMLFeedSpider which is the recommended spider to use -for scraping from an XML feed. - -See documentation in docs/topics/spiders.rst -""" - from __future__ import annotations from typing import TYPE_CHECKING, Any From 10ce3eaaf3220b51cf29928bd02b41a131f9a98a Mon Sep 17 00:00:00 2001 From: Rishi-gdrx Date: Sat, 4 Jul 2026 13:00:16 +0530 Subject: [PATCH 4/6] Remove duplicate module docstrings from utils, core, and misc modules. Fixes #7699 Co-authored-by: Cursor --- scrapy/__init__.py | 4 ---- scrapy/commands/__init__.py | 4 ---- scrapy/core/__init__.py | 3 --- scrapy/core/downloader/handlers/__init__.py | 2 -- scrapy/core/downloader/handlers/http11.py | 2 -- scrapy/core/engine.py | 7 ------- scrapy/core/scraper.py | 3 --- scrapy/link.py | 8 -------- scrapy/linkextractors/__init__.py | 8 -------- scrapy/linkextractors/lxmlhtml.py | 4 ---- scrapy/mail.py | 4 ---- scrapy/responsetypes.py | 5 ----- scrapy/selector/__init__.py | 4 ---- scrapy/squeues.py | 4 ---- scrapy/statscollectors.py | 4 ---- scrapy/utils/_download_handlers.py | 2 -- scrapy/utils/asyncio.py | 2 -- scrapy/utils/boto.py | 2 -- scrapy/utils/datatypes.py | 5 ----- scrapy/utils/defer.py | 4 ---- scrapy/utils/deprecate.py | 2 -- scrapy/utils/display.py | 4 ---- scrapy/utils/engine.py | 2 -- scrapy/utils/httpobj.py | 2 -- scrapy/utils/misc.py | 2 -- scrapy/utils/python.py | 4 ---- scrapy/utils/request.py | 5 ----- scrapy/utils/response.py | 5 ----- scrapy/utils/signal.py | 2 -- scrapy/utils/template.py | 2 -- scrapy/utils/test.py | 4 ---- scrapy/utils/url.py | 5 ----- 32 files changed, 120 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index c5b6483e3..03a7edf9a 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -1,7 +1,3 @@ -""" -Scrapy - a web crawling and web scraping framework written for Python -""" - import pkgutil import sys import warnings diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index d9c919db9..7388f07e2 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -1,7 +1,3 @@ -""" -Base class for Scrapy commands -""" - from __future__ import annotations import argparse diff --git a/scrapy/core/__init__.py b/scrapy/core/__init__.py index 59e07e643..e69de29bb 100644 --- a/scrapy/core/__init__.py +++ b/scrapy/core/__init__.py @@ -1,3 +0,0 @@ -""" -Scrapy core library classes and functions. -""" diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index fb27cdb8b..a2be49249 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,5 +1,3 @@ -"""Download handlers for different schemes""" - from __future__ import annotations import inspect diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 17dca5acb..a20b1975f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,5 +1,3 @@ -"""Download handlers for http and https schemes""" - from __future__ import annotations import ipaddress diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1033e874f..98bbec4ce 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -1,10 +1,3 @@ -""" -This is the Scrapy engine which controls the Scheduler, Downloader and Spider. - -For more information see docs/topics/architecture.rst - -""" - from __future__ import annotations import asyncio diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 58e37ce5e..d84b1af54 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,6 +1,3 @@ -"""This module implements the Scraper component which parses responses and -extracts information from them""" - from __future__ import annotations import logging diff --git a/scrapy/link.py b/scrapy/link.py index 8211faccd..936be9b31 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -1,11 +1,3 @@ -""" -This module defines the Link object used in Link extractors. - -For actual link extractors implementation see scrapy.linkextractors, or -its documentation in: docs/topics/link-extractors.rst -""" - - class Link: """Link objects represent an extracted link by the LinkExtractor. diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index b39859f7b..6620af6c1 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -1,11 +1,3 @@ -""" -scrapy.linkextractors - -This package contains a collection of Link Extractors. - -For more info see docs/topics/link-extractors.rst -""" - from __future__ import annotations from typing import TYPE_CHECKING diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 11fd62fb2..973ad1d6e 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,7 +1,3 @@ -""" -Link extractor based on lxml.html -""" - from __future__ import annotations import logging diff --git a/scrapy/mail.py b/scrapy/mail.py index 97123e63c..c8ced5f78 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -1,7 +1,3 @@ -""" -Mail sending helpers -""" - from __future__ import annotations import logging diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 29e9b6bb7..71960d28d 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -1,8 +1,3 @@ -""" -This module implements a class which returns the appropriate Response class -based on different criteria. -""" - from __future__ import annotations from io import StringIO diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 7cfa3c364..745772cc8 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -1,7 +1,3 @@ -""" -Selectors -""" - # top-level imports from scrapy.selector.unified import Selector, SelectorList diff --git a/scrapy/squeues.py b/scrapy/squeues.py index e06e38e23..b1354aed9 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -1,7 +1,3 @@ -""" -Scheduler queues -""" - from __future__ import annotations import marshal diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 1f9894519..f95edc349 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -1,7 +1,3 @@ -""" -Scrapy extension for collecting scraping stats -""" - from __future__ import annotations import logging diff --git a/scrapy/utils/_download_handlers.py b/scrapy/utils/_download_handlers.py index 23b715ac6..ce86b622e 100644 --- a/scrapy/utils/_download_handlers.py +++ b/scrapy/utils/_download_handlers.py @@ -1,5 +1,3 @@ -"""Utils for built-in HTTP download handlers.""" - from __future__ import annotations from contextlib import contextmanager diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 44604c0fe..e49215721 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -1,5 +1,3 @@ -"""Utilities related to asyncio and its support in Scrapy.""" - from __future__ import annotations import asyncio diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 73f86bc71..97a810388 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,5 +1,3 @@ -"""Boto/botocore helpers""" - def is_botocore_available() -> bool: try: diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index c020ff4b9..ff30c48b2 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -1,8 +1,3 @@ -""" -This module contains data types used by Scrapy which are not included in the -Python Standard Library. -""" - from __future__ import annotations import collections diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d0259b634..e736cad82 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -1,7 +1,3 @@ -""" -Helper functions for dealing with Twisted deferreds -""" - from __future__ import annotations import asyncio diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 4fd50fdad..8afa4f118 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -1,5 +1,3 @@ -"""Some helpers for deprecation messages""" - from __future__ import annotations import inspect diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 08e89dc0d..2528442e9 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -1,7 +1,3 @@ -""" -pprint and pformat wrappers with colorization support -""" - import ctypes import platform import sys diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 085720f66..73095626d 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -1,5 +1,3 @@ -"""Some debugging functions for working with the Scrapy engine""" - from __future__ import annotations # used in global tests code diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index 58b4539bf..dd73987cf 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,5 +1,3 @@ -"""Helper functions for scrapy.http objects (Request, Response)""" - from __future__ import annotations from typing import TYPE_CHECKING diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 20e7cb381..ab65ba853 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,5 +1,3 @@ -"""Helper functions which don't fit anywhere else""" - from __future__ import annotations import ast diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 00cf818b7..338daf015 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,7 +1,3 @@ -""" -This module contains essential stuff that should've come with Python itself ;) -""" - from __future__ import annotations import gc diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index b2fd9a834..05791c106 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -1,8 +1,3 @@ -""" -This module provides some useful functions for working with -scrapy.Request objects -""" - from __future__ import annotations import hashlib diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 7747a7b9b..64299fe73 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -1,8 +1,3 @@ -""" -This module provides some useful functions for working with -scrapy.http.Response objects -""" - from __future__ import annotations import os diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index eca95e225..79923a056 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,5 +1,3 @@ -"""Helper functions for working with signals""" - from __future__ import annotations import asyncio diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 977d5a42a..55c006b39 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,5 +1,3 @@ -"""Helper functions for working with templates""" - from __future__ import annotations import re diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 90ed70262..72b8d6b61 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -1,7 +1,3 @@ -""" -This module contains some assorted functions used in tests -""" - from __future__ import annotations import asyncio diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index f67853ece..3d8b6108b 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -1,8 +1,3 @@ -""" -This module contains general purpose URL functions not found in the standard -library. -""" - from __future__ import annotations import re From 78abe0013c90257afdda21ac9b3c1fc14ef8ea2e Mon Sep 17 00:00:00 2001 From: Rishi-gdrx Date: Sat, 4 Jul 2026 13:04:02 +0530 Subject: [PATCH 5/6] Move module docstring content to docs and remove remaining docstrings. Migrate contributor notes, FTP handler details, trackref PyPy behavior, SitemapSpider API stability warning, and STATSMAILER_RCPTS documentation into the official docs before removing the last module-level docstrings. Fixes #7699 Co-authored-by: Cursor --- docs/contributing.rst | 11 +++++++++ docs/topics/download-handlers.rst | 19 ++++++++++++++ docs/topics/exceptions.rst | 5 ++++ docs/topics/leaks.rst | 7 ++++++ docs/topics/request-response.rst | 2 ++ docs/topics/settings.rst | 12 +++++++++ docs/topics/signals.rst | 4 +++ docs/topics/spiders.rst | 5 ++++ scrapy/core/downloader/handlers/_httpx.py | 2 -- scrapy/core/downloader/handlers/ftp.py | 30 ----------------------- scrapy/exceptions.py | 7 ------ scrapy/extensions/statsmailer.py | 6 ----- scrapy/settings/default_settings.py | 15 ------------ scrapy/signals.py | 7 ------ scrapy/utils/sitemap.py | 7 ------ scrapy/utils/trackref.py | 14 ----------- 16 files changed, 65 insertions(+), 88 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index c868a0ac4..68cd1672c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -308,6 +308,17 @@ In any case, if something is covered in a docstring, use the documentation instead of duplicating the docstring in files within the ``docs/`` directory. +When adding a new setting to ``scrapy/settings/default_settings.py``: + +* add it in alphabetical order, with the exception that enabling flags and + other high-level settings for a group should come first in their group + and pairs like host/port and user/password should be in the usual order +* group similar settings without leaving blank lines +* add its documentation to :doc:`topics/settings` + +When adding a new exception or signal, document it in :doc:`topics/exceptions` +or :doc:`topics/signals`, respectively. + Documentation updates that cover new or modified features must use Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use ``VERSION`` as version, we will replace it with the actual version right before diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index c901e01e4..310b494fe 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -378,6 +378,25 @@ This handler supports ``ftp://host/path`` FTP URIs. It's implemented using :mod:`twisted.protocols.ftp`. +FTP connection parameters can be passed using :attr:`Request.meta +`: + +* ``ftp_user`` (falls back to :setting:`FTP_USER`) +* ``ftp_password`` (falls back to :setting:`FTP_PASSWORD`) +* ``ftp_passive`` (falls back to :setting:`FTP_PASSIVE_MODE`) +* ``ftp_local_filename``: if not given, file data is returned in + ``response.body``. If given, file data is saved to that local path (useful + for large files) and the local filename is also returned in ``response.body``. + +The built response uses HTTP-like status codes by default: ``200`` on success, +``404`` when the file is not found (FTP code 550), or the corresponding FTP +exception otherwise. The mapping from FTP return codes to response status codes +is defined in the :attr:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler.CODE_MAPPING` +class attribute. The ``default`` key is used for unmapped codes. + +On successful requests (status 200), ``response.headers`` includes +``Local Filename`` (when ``ftp_local_filename`` is used) and ``Size``. + S3DownloadHandler ----------------- diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index cc690d609..4508d6e60 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -7,6 +7,11 @@ Exceptions .. module:: scrapy.exceptions :synopsis: Scrapy exceptions +.. note:: + + When adding a new exception to ``scrapy.exceptions``, document it on this + page. + .. _topics-exceptions-ref: Built-in Exceptions reference diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 0913a3310..32286ab23 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -182,6 +182,13 @@ scrapy.utils.trackref module Here are the functions available in the :mod:`~scrapy.utils.trackref` module. +.. note:: + + PyPy uses a tracing garbage collector, so objects may remain in + ``live_refs`` longer than expected, even after they go out of scope. If + deterministic behavior is required, you may need to explicitly trigger + garbage collection or call ``trackref.live_refs.clear()``. + .. class:: object_ref Inherit from this class if you want to track live diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f2fcfb0b5..7c84df2f0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -720,6 +720,8 @@ Those are: * :reqmeta:`download_warnsize` * :reqmeta:`download_timeout` * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) +* ``ftp_passive`` (See :setting:`FTP_PASSIVE_MODE` for more info) +* ``ftp_local_filename`` (See :ref:`topics-download-handlers` for more info) * ``ftp_user`` (See :setting:`FTP_USER` for more info) * :reqmeta:`give_up_log_level` * :reqmeta:`handle_httpstatus_all` diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 11251a92c..005bedcc9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2056,6 +2056,18 @@ Example: .. note:: This is a :ref:`pre-crawler setting `. +.. setting:: STATSMAILER_RCPTS + +STATSMAILER_RCPTS +----------------- + +Default: ``[]`` + +A list of email addresses to receive a message with scraping stats when a +spider finishes. Used by the deprecated +:class:`~scrapy.extensions.statsmailer.StatsMailer` extension, which is enabled +when this setting is non-empty. + .. setting:: STATS_CLASS STATS_CLASS diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index d13733623..d0ed7110a 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -16,6 +16,10 @@ deliver the arguments that the handler receives. You can connect to signals (or send your own) through the :ref:`topics-api-signals`. +.. note:: + + When adding a new signal to ``scrapy.signals``, document it on this page. + Here is a simple example showing how you can catch signals and perform some action: .. code-block:: python diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 506daf930..11bce19cd 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -787,6 +787,11 @@ SitemapSpider SitemapSpider allows you to crawl a site by discovering the URLs using `Sitemaps`_. + .. note:: + + The :mod:`scrapy.utils.sitemap` module provides sitemap parsing support + for this spider. Its API is subject to change without notice. + It supports nested sitemaps and discovering sitemap urls from `robots.txt`_. diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index 9b54b44d8..c7aebf5c8 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -1,5 +1,3 @@ -"""``httpx``-based HTTP(S) download handler. Currently not recommended for production use.""" - from __future__ import annotations import ipaddress diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 07ff4a74e..633b263e8 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -1,33 +1,3 @@ -""" -An asynchronous FTP file download handler for scrapy which somehow emulates an http response. - -FTP connection parameters are passed using the request meta field: -- ftp_user (optional, falls back to FTP_USER) -- ftp_password (optional, falls back to FTP_PASSWORD) -- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode -- ftp_local_filename - - If not given, file data will come in the response.body, as a normal scrapy Response, - which will imply that the entire file will be on memory. - - if given, file data will be saved in a local file with the given name - This helps when downloading very big files to avoid memory issues. In addition, for - convenience the local file name will also be given in the response body. - -The status of the built html response will be, by default -- 200 in case of success -- 404 in case specified file was not found in the server (ftp code 550) - -or raise corresponding ftp exception otherwise - -The matching from server ftp command return codes to html response codes is defined in the -CODE_MAPPING attribute of the handler class. The key 'default' is used for any code -that is not explicitly present among the map keys. You may need to overwrite this -mapping if want a different behaviour than default. - -In case of status 200 request, response.headers will come with two keys: - 'Local Filename' - with the value of the local filename if given - 'Size' - with size of the downloaded data -""" - from __future__ import annotations import re diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 5330eab48..54a2cdbd6 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -1,10 +1,3 @@ -""" -Scrapy core exceptions - -These exceptions are documented in docs/topics/exceptions.rst. Please don't add -new exceptions here without documenting them there. -""" - from __future__ import annotations from typing import TYPE_CHECKING, Any diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index f05595806..25cc92dc6 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -1,9 +1,3 @@ -""" -StatsMailer extension sends an email when a spider finishes scraping. - -Use STATSMAILER_RCPTS setting to enable and give the recipient mail address -""" - from __future__ import annotations import warnings diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 049921ba7..76cc85f19 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -1,18 +1,3 @@ -"""This module contains the default values for all settings used by Scrapy. - -For more information about these settings you can read the settings -documentation in docs/topics/settings.rst - -Scrapy developers, if you add a setting here remember to: - -* add it in alphabetical order, with the exception that enabling flags and - other high-level settings for a group should come first in their group - and pairs like host/port and user/password should be in the usual order -* group similar settings without leaving blank lines -* add its documentation to the available settings documentation - (docs/topics/settings.rst) -""" - import sys from importlib import import_module from pathlib import Path diff --git a/scrapy/signals.py b/scrapy/signals.py index 972f4fd60..383ef0245 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -1,10 +1,3 @@ -""" -Scrapy signals - -These signals are documented in docs/topics/signals.rst. Please don't add new -signals here without documenting them there. -""" - engine_started = object() engine_stopped = object() scheduler_empty = object() diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 03b7bf3b1..2edf2d5ef 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -1,10 +1,3 @@ -""" -Module for processing Sitemaps. - -Note: The main purpose of this module is to provide support for the -SitemapSpider, its API is subject to change without notice. -""" - from __future__ import annotations import warnings diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 22f9eadd0..7822f99b7 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -1,17 +1,3 @@ -"""This module provides some functions and classes to record and report -references to live object instances. - -If you want live objects for a particular class to be tracked, you only have to -subclass from object_ref (instead of object). - -This library has a minimal performance impact. - -.. note:: PyPy uses a tracing garbage collector, so objects may - remain in the ``live_refs`` longer than expected, even after they - go out of scope. If deterministic behavior is required, you may need - to explicitly trigger garbage collection or call ``trackref.live_refs.clear()``. -""" - from __future__ import annotations from collections import defaultdict From 96437be1b52602e8e1b8592ff1dfcc743cc74c70 Mon Sep 17 00:00:00 2001 From: Rishi-gdrx Date: Tue, 7 Jul 2026 12:44:37 +0530 Subject: [PATCH 6/6] Address PR review feedback on docstring migration. Move contributor notes back to source comments, remove deprecated STATSMAILER_RCPTS docs, and fix the ftp_local_filename cross-reference. Co-authored-by: Cursor --- docs/contributing.rst | 11 ----------- docs/topics/download-handlers.rst | 2 ++ docs/topics/exceptions.rst | 5 ----- docs/topics/leaks.rst | 7 ------- docs/topics/request-response.rst | 2 +- docs/topics/settings.rst | 12 ------------ docs/topics/signals.rst | 4 ---- docs/topics/spiders.rst | 5 ----- scrapy/exceptions.py | 3 +++ scrapy/settings/default_settings.py | 8 ++++++++ scrapy/signals.py | 3 +++ scrapy/utils/sitemap.py | 3 +++ scrapy/utils/trackref.py | 5 +++++ 13 files changed, 25 insertions(+), 45 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 68cd1672c..c868a0ac4 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -308,17 +308,6 @@ In any case, if something is covered in a docstring, use the documentation instead of duplicating the docstring in files within the ``docs/`` directory. -When adding a new setting to ``scrapy/settings/default_settings.py``: - -* add it in alphabetical order, with the exception that enabling flags and - other high-level settings for a group should come first in their group - and pairs like host/port and user/password should be in the usual order -* group similar settings without leaving blank lines -* add its documentation to :doc:`topics/settings` - -When adding a new exception or signal, document it in :doc:`topics/exceptions` -or :doc:`topics/signals`, respectively. - Documentation updates that cover new or modified features must use Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use ``VERSION`` as version, we will replace it with the actual version right before diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 310b494fe..174237806 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -378,6 +378,8 @@ This handler supports ``ftp://host/path`` FTP URIs. It's implemented using :mod:`twisted.protocols.ftp`. +.. _topics-ftp-request-meta: + FTP connection parameters can be passed using :attr:`Request.meta `: diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 4508d6e60..cc690d609 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -7,11 +7,6 @@ Exceptions .. module:: scrapy.exceptions :synopsis: Scrapy exceptions -.. note:: - - When adding a new exception to ``scrapy.exceptions``, document it on this - page. - .. _topics-exceptions-ref: Built-in Exceptions reference diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 32286ab23..0913a3310 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -182,13 +182,6 @@ scrapy.utils.trackref module Here are the functions available in the :mod:`~scrapy.utils.trackref` module. -.. note:: - - PyPy uses a tracing garbage collector, so objects may remain in - ``live_refs`` longer than expected, even after they go out of scope. If - deterministic behavior is required, you may need to explicitly trigger - garbage collection or call ``trackref.live_refs.clear()``. - .. class:: object_ref Inherit from this class if you want to track live diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 7c84df2f0..6aa0f5845 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -721,7 +721,7 @@ Those are: * :reqmeta:`download_timeout` * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * ``ftp_passive`` (See :setting:`FTP_PASSIVE_MODE` for more info) -* ``ftp_local_filename`` (See :ref:`topics-download-handlers` for more info) +* ``ftp_local_filename`` (See :ref:`topics-ftp-request-meta` for more info) * ``ftp_user`` (See :setting:`FTP_USER` for more info) * :reqmeta:`give_up_log_level` * :reqmeta:`handle_httpstatus_all` diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 005bedcc9..11251a92c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2056,18 +2056,6 @@ Example: .. note:: This is a :ref:`pre-crawler setting `. -.. setting:: STATSMAILER_RCPTS - -STATSMAILER_RCPTS ------------------ - -Default: ``[]`` - -A list of email addresses to receive a message with scraping stats when a -spider finishes. Used by the deprecated -:class:`~scrapy.extensions.statsmailer.StatsMailer` extension, which is enabled -when this setting is non-empty. - .. setting:: STATS_CLASS STATS_CLASS diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index d0ed7110a..d13733623 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -16,10 +16,6 @@ deliver the arguments that the handler receives. You can connect to signals (or send your own) through the :ref:`topics-api-signals`. -.. note:: - - When adding a new signal to ``scrapy.signals``, document it on this page. - Here is a simple example showing how you can catch signals and perform some action: .. code-block:: python diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 11bce19cd..506daf930 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -787,11 +787,6 @@ SitemapSpider SitemapSpider allows you to crawl a site by discovering the URLs using `Sitemaps`_. - .. note:: - - The :mod:`scrapy.utils.sitemap` module provides sitemap parsing support - for this spider. Its API is subject to change without notice. - It supports nested sitemaps and discovering sitemap urls from `robots.txt`_. diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 54a2cdbd6..f611c4cff 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -1,3 +1,6 @@ +# These exceptions are documented in docs/topics/exceptions.rst. Please don't add +# new exceptions here without documenting them there. + from __future__ import annotations from typing import TYPE_CHECKING, Any diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 76cc85f19..4c7a209b2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -1,3 +1,11 @@ +# Scrapy developers, if you add a setting here remember to: +# +# * add it in alphabetical order, with the exception that enabling flags and +# other high-level settings for a group should come first in their group +# and pairs like host/port and user/password should be in the usual order +# * group similar settings without leaving blank lines +# * add its documentation to docs/topics/settings.rst + import sys from importlib import import_module from pathlib import Path diff --git a/scrapy/signals.py b/scrapy/signals.py index 383ef0245..81c737883 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -1,3 +1,6 @@ +# These signals are documented in docs/topics/signals.rst. Please don't add new +# signals here without documenting them there. + engine_started = object() engine_stopped = object() scheduler_empty = object() diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 2edf2d5ef..11e8e98ef 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -1,3 +1,6 @@ +# The main purpose of this module is to provide support for the SitemapSpider, +# its API is subject to change without notice. + from __future__ import annotations import warnings diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 7822f99b7..8bb5c4fa3 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -1,3 +1,8 @@ +# PyPy uses a tracing garbage collector, so objects may remain in live_refs +# longer than expected, even after they go out of scope. If deterministic +# behavior is required, you may need to explicitly trigger garbage collection +# or call trackref.live_refs.clear(). + from __future__ import annotations from collections import defaultdict