mirror of https://github.com/scrapy/scrapy.git
20 lines
705 B
Python
20 lines
705 B
Python
"""Helper functions for scrapy.http objects (Request, Response)"""
|
|
|
|
from typing import Union
|
|
from urllib.parse import urlparse, ParseResult
|
|
from weakref import WeakKeyDictionary
|
|
|
|
from scrapy.http import Request, Response
|
|
|
|
|
|
_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary()
|
|
|
|
|
|
def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult:
|
|
"""Return urlparse.urlparse caching the result, where the argument can be a
|
|
Request or Response object
|
|
"""
|
|
if request_or_response not in _urlparse_cache:
|
|
_urlparse_cache[request_or_response] = urlparse(request_or_response.url)
|
|
return _urlparse_cache[request_or_response]
|