From 751b384b87642d04aafdaee745dad6fea66ca7c1 Mon Sep 17 00:00:00 2001 From: Rishi-gdrx Date: Sat, 4 Jul 2026 12:54:53 +0530 Subject: [PATCH] Remove duplicate module docstrings from middleware modules. Fixes #7699 Co-authored-by: Cursor --- scrapy/core/downloader/middleware.py | 6 ------ scrapy/core/spidermw.py | 6 ------ scrapy/downloadermiddlewares/defaultheaders.py | 6 ------ scrapy/downloadermiddlewares/downloadtimeout.py | 6 ------ scrapy/downloadermiddlewares/httpauth.py | 6 ------ scrapy/downloadermiddlewares/retry.py | 9 --------- scrapy/downloadermiddlewares/robotstxt.py | 6 ------ scrapy/downloadermiddlewares/useragent.py | 2 -- scrapy/spidermiddlewares/depth.py | 6 ------ scrapy/spidermiddlewares/httperror.py | 6 ------ scrapy/spidermiddlewares/referer.py | 5 ----- scrapy/spidermiddlewares/urllength.py | 6 ------ 12 files changed, 70 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index ab74e22a4..2f811cac0 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -1,9 +1,3 @@ -""" -Downloader Middleware manager - -See documentation in docs/topics/downloader-middleware.rst -""" - from __future__ import annotations import warnings diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index fbc6f2530..bf31cb454 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -1,9 +1,3 @@ -""" -Spider Middleware manager - -See documentation in docs/topics/spider-middleware.rst -""" - from __future__ import annotations import logging diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 7c940a078..5aa8f37d7 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -1,9 +1,3 @@ -""" -DefaultHeaders downloader middleware - -See documentation in docs/topics/downloader-middleware.rst -""" - from __future__ import annotations from typing import TYPE_CHECKING diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index bccfb230c..79d0aebdf 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -1,9 +1,3 @@ -""" -Download timeout middleware - -See documentation in docs/topics/downloader-middleware.rst -""" - from __future__ import annotations from typing import TYPE_CHECKING diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index b09abdadd..2eaa9e532 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -1,9 +1,3 @@ -""" -HTTP basic auth downloader middleware - -See documentation in docs/topics/downloader-middleware.rst -""" - from __future__ import annotations import warnings diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index a0a0b60a2..6f2fe63e4 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -1,12 +1,3 @@ -""" -An extension to retry failed requests that are potentially caused by temporary -problems such as a connection timeout or HTTP 500 error. - -You can change the behaviour of this middleware by modifying the scraping settings: -RETRY_TIMES - how many times to retry a failed page -RETRY_HTTP_CODES - which HTTP response codes to retry -""" - from __future__ import annotations from logging import Logger, getLevelName, getLogger diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7d0c17884..4b125c00a 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -1,9 +1,3 @@ -""" -This is a middleware to respect robots.txt policies. To activate it you must -enable this middleware and enable the ROBOTSTXT_OBEY setting. - -""" - from __future__ import annotations import logging diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 61f84b518..a7438fe76 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -1,5 +1,3 @@ -"""Set User-Agent header per spider or use a default value from settings""" - from __future__ import annotations from typing import TYPE_CHECKING diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 9c387530c..9179c97de 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -1,9 +1,3 @@ -""" -Depth Spider Middleware - -See documentation in docs/topics/spider-middleware.rst -""" - from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 156b73e7e..009704833 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -1,9 +1,3 @@ -""" -HttpError Spider Middleware - -See documentation in docs/topics/spider-middleware.rst -""" - from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 1305874d5..42b036a95 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -1,8 +1,3 @@ -""" -RefererMiddleware: populates Request referer field, based on the Response which -originated it. -""" - from __future__ import annotations import warnings diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index f325ce7a0..a938cd69e 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -1,9 +1,3 @@ -""" -Url Length Spider Middleware - -See documentation in docs/topics/spider-middleware.rst -""" - from __future__ import annotations import logging