From a0d1ad20013cd24da5aeba2c7742a2719d83f599 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 18:15:27 +0200 Subject: [PATCH] Document that follow_all requests share meta and cb_kwargs values (#7962) --- scrapy/http/response/__init__.py | 5 +++++ scrapy/http/response/text.py | 3 +++ scrapy/spidermiddlewares/metacopy.py | 3 +-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f1db11488..f91cb2095 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -305,6 +305,11 @@ class Response(object_ref): :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all` method which supports selectors in addition to absolute/relative URLs and Link objects. + + .. caution:: Every returned request gets its own *meta* and + *cb_kwargs* dictionaries, but the values within them are shared. + Mutating one of those values, e.g. appending to a list, affects + all the returned requests. """ if not hasattr(urls, "__iter__"): raise TypeError("'urls' argument must be an iterable") diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 64251780a..a8e452f7e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -276,6 +276,9 @@ class TextResponse(Response): using the ``css`` or ``xpath`` parameters, this method will not produce requests for selectors from which links cannot be obtained (for instance, anchor tags without an ``href`` attribute) + + .. seealso:: :meth:`.Response.follow_all`, for a caution about mutable + *meta* and *cb_kwargs* values. """ arguments = [x for x in (urls, css, xpath) if x is not None] if len(arguments) != 1: diff --git a/scrapy/spidermiddlewares/metacopy.py b/scrapy/spidermiddlewares/metacopy.py index aa5a120c6..2bef974d6 100644 --- a/scrapy/spidermiddlewares/metacopy.py +++ b/scrapy/spidermiddlewares/metacopy.py @@ -15,8 +15,7 @@ logger = logging.getLogger(__name__) class MetaCopyDetectionMiddleware(BaseSpiderMiddleware): """Warn when a spider yields a request with internal meta keys that should - not be copied from response.meta, or when two requests share the same meta - dict object. + not be copied from response.meta. Each warning is emitted at most once per crawl. """