diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index e0501c169..a160f1ee6 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -43,6 +43,44 @@ for additional schemes and to replace or disable default ones: :ref:`security-local-resources`, for the security implications of the default ``http``, ``ftp``, ``file`` and ``data`` handlers. +.. _download-handler-ids: + +Choosing a download handler per request +--------------------------------------- + +.. versionadded:: VERSION + +The :setting:`DOWNLOAD_HANDLERS_BY_NAME` setting registers handlers under a +name instead of a URL scheme. Such handlers are only used by requests that ask +for them through the :reqmeta:`download_handler` metadata key: + +.. code-block:: python + + DOWNLOAD_HANDLERS_BY_NAME = { + "playwright": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler", + } + +.. code-block:: python + + Request("https://example.com", meta={"download_handler": "playwright"}) + +:reqmeta:`download_handler` also accepts a URL scheme, e.g. ``"https"``, to use +a handler from :setting:`DOWNLOAD_HANDLERS` on a request whose URL uses a +different scheme. + +Requests keep their :reqmeta:`download_handler` across redirects and retries, +and :ref:`robots.txt ` requests inherit it from the request +that triggers them. To fetch ``robots.txt`` differently, use a :ref:`downloader +middleware ` that acts on requests with the +:reqmeta:`is_robotstxt_request` metadata key: + +.. code-block:: python + + class DirectRobotsTxtMiddleware: + def process_request(self, request): + if request.meta.get("is_robotstxt_request"): + request.meta.pop("download_handler", None) + Replacing HTTP(S) download handlers ----------------------------------- diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 75158440b..6d5f06100 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -943,6 +943,17 @@ This meta key is not supported by :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`, but the :setting:`DOWNLOAD_BIND_ADDRESS` is supported by it. +.. reqmeta:: download_handler + +download_handler +---------------- + +.. versionadded:: VERSION + +ID of the :ref:`download handler ` to use for this +request, either a name from :setting:`DOWNLOAD_HANDLERS_BY_NAME` or a URL +scheme from :setting:`DOWNLOAD_HANDLERS`. See :ref:`download-handler-ids`. + .. reqmeta:: download_timeout download_timeout @@ -1006,6 +1017,18 @@ http_user Overrides :setting:`HTTPAUTH_USER` for this request. +.. reqmeta:: is_robotstxt_request + +is_robotstxt_request +-------------------- + +.. versionadded:: VERSION + +``True`` on the ``robots.txt`` requests that +:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` sends, so +that :ref:`downloader middlewares ` can tell them +apart from other requests. + .. reqmeta:: max_retry_times max_retry_times diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 65ee77258..3595150f4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1062,6 +1062,19 @@ handler (without replacement), place this in your ``settings.py``: :ref:`security-local-resources` +.. setting:: DOWNLOAD_HANDLERS_BY_NAME + +DOWNLOAD_HANDLERS_BY_NAME +------------------------- + +.. versionadded:: VERSION + +Default: ``{}`` + +A dict mapping names to :ref:`download handlers ` +that requests can ask for by name. See :ref:`download-handler-ids`. + + .. setting:: DOWNLOAD_SLOTS DOWNLOAD_SLOTS diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 84dc6216b..ea1af22dc 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -61,44 +61,59 @@ class DownloadHandlerProtocol(Protocol): class DownloadHandlers: def __init__(self, crawler: Crawler): self._crawler: Crawler = crawler - # stores acceptable schemes on instancing - self._schemes: dict[str, str | Callable[..., Any]] = {} - # stores instanced handlers for schemes + # stores class paths by handler ID + self._paths: dict[str, str | Callable[..., Any]] = {} + # handler IDs that are also acceptable URL schemes + self._schemes: set[str] = set() + # stores instanced handlers by handler ID self._handlers: dict[str, DownloadHandlerProtocol] = {} # remembers failed handlers self._notconfigured: dict[str, str] = {} # remembers handlers with Deferred-based download_request() self._old_style_handlers: set[str] = set() - handlers: dict[str, str | Callable[..., Any]] = without_none_values( + scheme_handlers: dict[str, str | Callable[..., Any]] = without_none_values( cast( "dict[str, str | Callable[..., Any]]", crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), ) ) - for scheme, clspath in handlers.items(): - self._schemes[scheme] = clspath - self._load_handler(scheme, skip_lazy=True) + named_handlers: dict[str, str | Callable[..., Any]] = without_none_values( + cast( + "dict[str, str | Callable[..., Any]]", + crawler.settings.getdict("DOWNLOAD_HANDLERS_BY_NAME"), + ) + ) + if clashes := scheme_handlers.keys() & named_handlers.keys(): + raise ValueError( + f"The following download handler IDs are defined both in " + f"DOWNLOAD_HANDLERS and in DOWNLOAD_HANDLERS_BY_NAME: " + f"{', '.join(sorted(clashes))}." + ) + self._schemes.update(scheme_handlers) + for handler_id, clspath in (scheme_handlers | named_handlers).items(): + self._paths[handler_id] = clspath + self._load_handler(handler_id, skip_lazy=True) crawler.signals.connect(self._close, signals.engine_stopped) - def _get_handler(self, scheme: str) -> DownloadHandlerProtocol | None: - """Lazy-load the downloadhandler for a scheme - only on the first request for that scheme. + def _get_handler(self, handler_id: str) -> DownloadHandlerProtocol | None: + """Lazy-load the download handler with the given ID only on its first + request. """ - if scheme in self._handlers: - return self._handlers[scheme] - if scheme in self._notconfigured: + if handler_id in self._handlers: + return self._handlers[handler_id] + if handler_id in self._notconfigured: return None - if scheme not in self._schemes: - self._notconfigured[scheme] = "no handler available for that scheme" + if handler_id not in self._paths: + self._notconfigured[handler_id] = "no handler with that ID" return None - return self._load_handler(scheme) + return self._load_handler(handler_id) def _load_handler( - self, scheme: str, skip_lazy: bool = False + self, handler_id: str, skip_lazy: bool = False ) -> DownloadHandlerProtocol | None: - path = self._schemes[scheme] + path = self._paths[handler_id] try: dhcls: type[DownloadHandlerProtocol] = load_object(path) if skip_lazy: @@ -117,18 +132,18 @@ class DownloadHandlers: self._crawler, ) except NotConfigured as ex: - self._notconfigured[scheme] = str(ex) + self._notconfigured[handler_id] = str(ex) return None except Exception as ex: logger.error( - 'Loading "%(clspath)s" for scheme "%(scheme)s"', - {"clspath": path, "scheme": scheme}, + 'Loading "%(clspath)s" for handler ID "%(handler_id)s"', + {"clspath": path, "handler_id": handler_id}, exc_info=True, extra={"crawler": self._crawler}, ) - self._notconfigured[scheme] = str(ex) + self._notconfigured[handler_id] = str(ex) return None - self._handlers[scheme] = dh + self._handlers[handler_id] = dh if not inspect.iscoroutinefunction(dh.download_request): # pragma: no cover warnings.warn( f"{global_object_name(dh.download_request)} is not a coroutine function." @@ -137,7 +152,7 @@ class DownloadHandlers: category=ScrapyDeprecationWarning, stacklevel=1, ) - self._old_style_handlers.add(scheme) + self._old_style_handlers.add(handler_id) return dh def download_request( @@ -151,14 +166,33 @@ class DownloadHandlers: return deferred_from_coro(self.download_request_async(request)) async def download_request_async(self, request: Request) -> Response: - scheme = urlparse_cached(request).scheme - handler = self._get_handler(scheme) - if not handler: - raise NotSupported( - f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}" - ) + handler_id = request.meta.get("download_handler") + if handler_id is None: + scheme = urlparse_cached(request).scheme + # Handler IDs from DOWNLOAD_HANDLERS_BY_NAME are only reachable + # through the download_handler request metadata key, so that + # registering one does not make a new URL scheme downloadable. + if scheme in self._schemes: + handler = self._get_handler(scheme) + else: + handler = None + self._notconfigured.setdefault( + scheme, "no handler available for that scheme" + ) + if not handler: + raise NotSupported( + f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}" + ) + handler_id = scheme + else: + handler = self._get_handler(handler_id) + if not handler: + raise NotSupported( + f"Unusable download handler {handler_id!r}: " + f"{self._notconfigured[handler_id]}" + ) assert self._crawler.spider - if scheme in self._old_style_handlers: # pragma: no cover + if handler_id in self._old_style_handlers: # pragma: no cover return await maybe_deferred_to_future( cast( "Deferred[Response]", diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 81a3a887f..c5bde137f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from twisted.internet.defer import Deferred @@ -89,10 +89,16 @@ class RobotsTxtMiddleware: if netloc not in self._parsers: self._parsers[netloc] = Deferred() robotsurl = f"{url.scheme}://{url.netloc}/robots.txt" + meta: dict[str, Any] = { + "dont_obey_robotstxt": True, + "is_robotstxt_request": True, + } + if "download_handler" in request.meta: + meta["download_handler"] = request.meta["download_handler"] robotsreq = Request( robotsurl, priority=self.DOWNLOAD_PRIORITY, - meta={"dont_obey_robotstxt": True}, + meta=meta, callback=NO_CALLBACK, ) assert self.crawler.engine diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a44b36c8a..1e62dae32 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -71,6 +71,7 @@ __all__ = [ "DOWNLOAD_FAIL_ON_DATALOSS", "DOWNLOAD_HANDLERS", "DOWNLOAD_HANDLERS_BASE", + "DOWNLOAD_HANDLERS_BY_NAME", "DOWNLOAD_MAXSIZE", "DOWNLOAD_SLOTS", "DOWNLOAD_TIMEOUT", @@ -292,6 +293,7 @@ DOWNLOAD_HANDLERS_BASE = { "s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler", "ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler", } +DOWNLOAD_HANDLERS_BY_NAME = {} DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index b2fd9a834..68da71d15 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -32,7 +32,7 @@ logger = logging.getLogger(__name__) _fingerprint_cache: WeakKeyDictionary[ - Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool, str | None], bytes] ] = WeakKeyDictionary() @@ -77,8 +77,18 @@ def fingerprint( ) verbatim_url = bool(request.meta.get("verbatim_url")) effective_keep_fragments = keep_fragments and not verbatim_url + # A handler ID matching the URL scheme is the one that would be used + # anyway, so it is left out to keep such fingerprints unchanged. + handler_id: str | None = request.meta.get("download_handler") + if handler_id == urlparse_cached(request).scheme: + handler_id = None cache = _fingerprint_cache.setdefault(request, {}) - cache_key = (processed_include_headers, effective_keep_fragments, verbatim_url) + cache_key = ( + processed_include_headers, + effective_keep_fragments, + verbatim_url, + handler_id, + ) if cache_key not in cache: # To decode bytes reliably (JSON does not support bytes), regardless of # character encoding, we use bytes.hex() @@ -100,6 +110,8 @@ def fingerprint( "body": (request.body or b"").hex(), "headers": headers, } + if handler_id is not None: + fingerprint_data["download_handler"] = handler_id fingerprint_json = json.dumps(fingerprint_data, sort_keys=True) cache[cache_key] = hashlib.sha1( # noqa: S324 fingerprint_json.encode() @@ -114,13 +126,17 @@ class RequestFingerprinterProtocol(Protocol): class RequestFingerprinter: """Default fingerprinter. + .. versionchanged:: VERSION + :reqmeta:`download_handler` is taken into account. + It takes into account a canonical version (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url ` and the values of :attr:`request.method ` and :attr:`request.body `, unless :reqmeta:`verbatim_url` is true for that - request. It then generates an `SHA1 `_ - hash. + request. It also takes into account :reqmeta:`download_handler` when it + does not match the URL scheme. It then generates an `SHA1 + `_ hash. """ @classmethod diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8cb9308ff..2ba420294 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -15,8 +15,8 @@ from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Request, TextResponse +from scrapy.exceptions import NotConfigured, NotSupported, ScrapyDeprecationWarning +from scrapy.http import Request, Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.utils.boto import is_botocore_available from scrapy.utils.misc import build_from_crawler @@ -31,6 +31,20 @@ class DummyDH: pass +class NamedDH: + lazy = False + + async def download_request(self, request): + return Response(request.url, body=b"named") + + +class SchemeDH: + lazy = False + + async def download_request(self, request): + return Response(request.url, body=b"scheme") + + class DummyLazyDH: # Default (but deprecated) is lazy for backward compatibility async def download_request(self, request): @@ -84,7 +98,7 @@ class TestLoad: assert "scheme" not in dh._handlers assert "scheme" in dh._notconfigured assert ( - 'Loading "" for scheme "scheme"' + 'Loading "" for handler ID "scheme"' in caplog.text ) @@ -115,6 +129,58 @@ class TestLoad: assert "scheme" not in dh._notconfigured +class TestHandlerID: + @staticmethod + def _get_dh() -> DownloadHandlers: + crawler = get_crawler( + settings_dict={ + "DOWNLOAD_HANDLERS": {"https": SchemeDH}, + "DOWNLOAD_HANDLERS_BY_NAME": {"named": NamedDH}, + } + ) + crawler.spider = crawler._create_spider() + return DownloadHandlers(crawler) + + def test_clashing_ids(self) -> None: + crawler = get_crawler( + settings_dict={ + "DOWNLOAD_HANDLERS": {"https": SchemeDH}, + "DOWNLOAD_HANDLERS_BY_NAME": {"https": NamedDH}, + } + ) + with pytest.raises(ValueError, match="defined both in DOWNLOAD_HANDLERS"): + DownloadHandlers(crawler) + + @coroutine_test + async def test_name_not_a_scheme(self) -> None: + dh = self._get_dh() + with pytest.raises(NotSupported, match="Unsupported URL scheme 'named'"): + await dh.download_request_async(Request("named://example.com")) + + @coroutine_test + async def test_name_in_meta(self) -> None: + dh = self._get_dh() + request = Request("https://example.com", meta={"download_handler": "named"}) + response = await dh.download_request_async(request) + assert response.body == b"named" + + @coroutine_test + async def test_scheme_in_meta(self) -> None: + dh = self._get_dh() + request = Request("ftp://example.com", meta={"download_handler": "https"}) + response = await dh.download_request_async(request) + assert response.body == b"scheme" + + @coroutine_test + async def test_unknown_id_in_meta(self) -> None: + dh = self._get_dh() + request = Request("https://example.com", meta={"download_handler": "nmaed"}) + with pytest.raises( + NotSupported, match="Unusable download handler 'nmaed': no handler with" + ): + await dh.download_request_async(request) + + class TestFile: def setup_method(self): # add a special char to check that they are handled correctly diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 793a2b5be..8ccd6e05a 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -245,6 +245,17 @@ Disallow: /some/randome/page.html await middleware.process_request(Request("http://site.local/allowed")) assert middleware.process_request_2.called + @coroutine_test + async def test_robotstxt_download_handler(self) -> None: + middleware = RobotsTxtMiddleware(self._get_successful_crawler()) + await self.assertNotIgnored( + Request("http://site.local/allowed", meta={"download_handler": "named"}), + middleware, + ) + robotsreq = self.crawler.engine.download_async.call_args_list[0][0][0] + assert robotsreq.meta["download_handler"] == "named" + assert robotsreq.meta["is_robotstxt_request"] is True + async def assertNotIgnored( self, request: Request, middleware: RobotsTxtMiddleware ) -> None: diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 935447bc4..56eeaa70d 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -73,13 +73,15 @@ class TestFingerprint: function: _FingerprintFunction = staticmethod(fingerprint) cache: ( WeakKeyDictionary[ - Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] + Request, + dict[tuple[tuple[bytes, ...] | None, bool, bool, str | None], bytes], ] | WeakKeyDictionary[ - Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], str] + Request, + dict[tuple[tuple[bytes, ...] | None, bool, bool, str | None], str], ] ) = _fingerprint_cache - default_cache_key = (None, False, False) + default_cache_key = (None, False, False, None) known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = ( ( Request("http://example.org"), @@ -235,6 +237,39 @@ class TestFingerprint: assert self.function(r5) == self.function(r5, keep_fragments=True) assert self.function(r5) == self.function(r5, keep_fragments=False) + def test_download_handler(self): + r1 = Request("https://example.com") + r2 = Request("https://example.com", meta={"download_handler": "playwright"}) + r3 = Request("https://example.com", meta={"download_handler": "http"}) + assert self.function(r1) != self.function(r2) + assert self.function(r2) != self.function(r3) + + # An ID matching the URL scheme is the handler that would be used + # anyway, so it does not change the fingerprint. + r4 = Request("https://example.com", meta={"download_handler": "https"}) + assert self.function(r1) == self.function(r4) + + # IDs are case-sensitive, and URL schemes are lowercased. + r5 = Request("HTTPS://example.com", meta={"download_handler": "HTTPS"}) + assert self.function(r1) != self.function(r5) + + def test_download_handler_caching(self): + # The cached fingerprint must account for the handler ID, which can + # change after the first call. + r1 = Request("https://example.com") + fp1 = self.function(r1) + r1.meta["download_handler"] = "playwright" + assert self.function(r1) != fp1 + + def test_download_handler_and_verbatim_url(self): + r1 = Request( + "https://example.com/a b", + meta={"verbatim_url": True, "download_handler": "playwright"}, + ) + r2 = Request("https://example.com/a b", meta={"verbatim_url": True}) + r3 = Request("https://example.com/a b", meta={"download_handler": "playwright"}) + assert len({self.function(r1), self.function(r2), self.function(r3)}) == 3 + def test_method_and_body(self): r1 = Request("http://www.example.com") r2 = Request("http://www.example.com", method="POST")