From 58ed9fdcccc9d8a3c278a0bd0af54ff77863b034 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 24 Jul 2026 17:08:58 +0200 Subject: [PATCH] Document urlparse_cached (#7777) --- docs/topics/request-response.rst | 2 ++ scrapy/utils/httpobj.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 7468f1b64..b83a04032 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -342,6 +342,8 @@ Other functions related to requests .. autofunction:: scrapy.utils.request.request_from_dict +.. autofunction:: scrapy.utils.httpobj.urlparse_cached + .. _topics-request-response-ref-request-callback-arguments: diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index 58b4539bf..5965e1f21 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -16,8 +16,15 @@ _urlparse_cache: WeakKeyDictionary[Request | Response, ParseResult] = ( def urlparse_cached(request_or_response: Request | Response) -> ParseResult: - """Return urlparse.urlparse caching the result, where the argument can be a - Request or Response object + """Return the result of parsing the URL of *request_or_response*, a + :class:`~scrapy.Request` or :class:`~scrapy.http.Response` object, with + :func:`urllib.parse.urlparse`. + + The result is cached, using a :class:`weakref.WeakKeyDictionary` keyed on + *request_or_response*, so that the URL of a given object is parsed only + once. Prefer this function over calling :func:`urllib.parse.urlparse` on + ``request_or_response.url`` directly when the same URL may be parsed more + than once. """ if request_or_response not in _urlparse_cache: _urlparse_cache[request_or_response] = urlparse(request_or_response.url)