RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Document urlparse_cached (#7777)

This commit is contained in:
Adrian 2026-07-24 17:08:58 +02:00 committed by GitHub
parent 41bb09741a
commit 58ed9fdccc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 11 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/topics/request-response.rst
View File

@ -342,6 +342,8 @@ Other functions related to requests
.. autofunction:: scrapy.utils.request.request_from_dict
.. autofunction:: scrapy.utils.httpobj.urlparse_cached
.. _topics-request-response-ref-request-callback-arguments:

11
scrapy/utils/httpobj.py
View File

@ -16,8 +16,15 @@ _urlparse_cache: WeakKeyDictionary[Request | Response, ParseResult] = (
def urlparse_cached(request_or_response: Request | Response) -> ParseResult:
"""Return urlparse.urlparse caching the result, where the argument can be a
Request or Response object
"""Return the result of parsing the URL of *request_or_response*, a
:class:`~scrapy.Request` or :class:`~scrapy.http.Response` object, with
:func:`urllib.parse.urlparse`.
The result is cached, using a :class:`weakref.WeakKeyDictionary` keyed on
*request_or_response*, so that the URL of a given object is parsed only
once. Prefer this function over calling :func:`urllib.parse.urlparse` on
``request_or_response.url`` directly when the same URL may be parsed more
than once.
"""
if request_or_response not in _urlparse_cache:
_urlparse_cache[request_or_response] = urlparse(request_or_response.url)